Refactor ARM monitor to use oci_common module

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-03 02:55:55 +08:00
parent 043924b747
commit c37ddae932
2 changed files with 143 additions and 73 deletions
@@ -0,0 +1,118 @@
+import oci
+import time
+import smtplib
+import logging
+import sys
+import os
+from email.mime.text import MIMEText
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv(os.path.join(os.path.dirname(__file__), ".env"))
+
+# Constants from environment
+EMAIL_TO = os.environ.get("EMAIL_TO")
+EMAIL_FROM = os.environ.get("EMAIL_FROM")
+EMAIL_APP_PASSWORD = os.environ.get("EMAIL_APP_PASSWORD")
+
+def get_logger(name, log_file=None):
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+
+    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+
+    # Console handler
+    ch = logging.StreamHandler()
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+
+    # File handler
+    if log_file:
+        fh = logging.FileHandler(log_file)
+        fh.setFormatter(formatter)
+        logger.addHandler(fh)
+
+    return logger
+
+def send_email(subject, body, logger=None):
+    if not all([EMAIL_FROM, EMAIL_TO, EMAIL_APP_PASSWORD]):
+        if logger:
+            logger.error("Email configuration missing. Cannot send email.")
+        return
+
+    msg = MIMEText(body)
+    msg["Subject"] = subject
+    msg["From"] = EMAIL_FROM
+    msg["To"] = EMAIL_TO
+
+    try:
+        with smtplib.SMTP_SSL("smtp.gmail.com", 465) as s:
+            s.login(EMAIL_FROM, EMAIL_APP_PASSWORD)
+            s.send_message(msg)
+    except Exception as e:
+        if logger:
+            logger.error(f"Failed to send email: {e}")
+
+def run_monitor(name, shape, launch_details_func, interval=60, weekly_report_interval=7*24*3600):
+    logger = get_logger(name, f"{name.lower()}.log")
+    logger.info(f"Starting {name} instance monitor ({shape})...")
+
+    config = oci.config.from_file()
+    compute = oci.core.ComputeClient(config)
+
+    start_time = time.time()
+    last_report = start_time
+    attempts = 0
+
+    while True:
+        try:
+            launch_details = launch_details_func()
+            resp = compute.launch_instance(launch_details)
+
+            instance_id = resp.data.id
+            msg = f"{name} instance created!\nID: {instance_id}"
+            logger.info(msg)
+            send_email(f"OCI {name} Instance Created!", msg, logger)
+            sys.exit(0)
+
+        except oci.exceptions.ServiceError as e:
+            msg_lower = str(e.message).lower()
+            attempts += 1
+
+            if "capacity" in msg_lower or "host" in msg_lower or e.status >= 500:
+                reason = "No capacity" if "capacity" in msg_lower else e.message
+                logger.info(f"{reason}. Retrying in {interval}s...")
+            elif "too many requests" in msg_lower or e.status == 429:
+                logger.warning(f"Rate limited. Retrying in {interval}s...")
+            elif "limit" in msg_lower:
+                logger.warning(f"Limit reached: {e.message}. Retrying in {interval}s...")
+            else:
+                logger.error(f"Unexpected OCI error: {e.message}")
+                send_email(f"OCI {name} Monitor Error", str(e.message), logger)
+                sys.exit(1)
+
+        except Exception as e:
+            attempts += 1
+            error_msg = str(e)
+            logger.error(f"Unexpected error: {error_msg}")
+            if "too many requests" not in error_msg.lower():
+                # For non-rate-limit unexpected errors, we might want to know, but keep retrying
+                pass
+            logger.info(f"Retrying in {interval}s...")
+
+        # Weekly Report
+        now = time.time()
+        if now - last_report >= weekly_report_interval:
+            uptime_days = int((now - start_time) / 86400)
+            report_body = (
+                f"Status: Still running, no capacity available.\n"
+                f"Uptime: {uptime_days} days\n"
+                f"Attempts since last report: {attempts}\n"
+                f"Shape: {shape}"
+            )
+            send_email(f"OCI {name} Monitor - Weekly Report", report_body, logger)
+            logger.info("Weekly report sent.")
+            last_report = now
+            attempts = 0
+
+        time.sleep(interval)