import oci import time import smtplib import logging import sys import os from email.mime.text import MIMEText from dotenv import load_dotenv # Load environment variables load_dotenv(os.path.join(os.path.dirname(__file__), ".env")) # Constants from environment EMAIL_TO = os.environ.get("EMAIL_TO") EMAIL_FROM = os.environ.get("EMAIL_FROM") EMAIL_APP_PASSWORD = os.environ.get("EMAIL_APP_PASSWORD") def get_logger(name, log_file=None): logger = logging.getLogger(name) logger.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') # Console handler ch = logging.StreamHandler() ch.setFormatter(formatter) logger.addHandler(ch) # File handler if log_file: fh = logging.FileHandler(log_file) fh.setFormatter(formatter) logger.addHandler(fh) return logger def send_email(subject, body, logger=None): if not all([EMAIL_FROM, EMAIL_TO, EMAIL_APP_PASSWORD]): if logger: logger.error("Email configuration missing. Cannot send email.") return msg = MIMEText(body) msg["Subject"] = subject msg["From"] = EMAIL_FROM msg["To"] = EMAIL_TO try: with smtplib.SMTP_SSL("smtp.gmail.com", 465) as s: s.login(EMAIL_FROM, EMAIL_APP_PASSWORD) s.send_message(msg) except Exception as e: if logger: logger.error(f"Failed to send email: {e}") def run_monitor(name, shape, launch_details_func, interval=60, weekly_report_interval=7*24*3600): logger = get_logger(name, f"{name.lower()}.log") logger.info(f"Starting {name} instance monitor ({shape})...") config = oci.config.from_file() compute = oci.core.ComputeClient(config) start_time = time.time() last_report = start_time attempts = 0 while True: try: launch_details = launch_details_func() resp = compute.launch_instance(launch_details) instance_id = resp.data.id msg = f"{name} instance created!\nID: {instance_id}" logger.info(msg) send_email(f"OCI {name} Instance Created!", msg, logger) sys.exit(0) except oci.exceptions.ServiceError as e: msg_lower = str(e.message).lower() attempts += 1 if "capacity" in msg_lower or "host" in msg_lower or e.status >= 500: reason = "No capacity" if "capacity" in msg_lower else e.message logger.info(f"{reason}. Retrying in {interval}s...") elif "too many requests" in msg_lower or e.status == 429: logger.warning(f"Rate limited. Retrying in {interval}s...") elif "limit" in msg_lower: logger.warning(f"Limit reached: {e.message}. Retrying in {interval}s...") else: logger.error(f"Unexpected OCI error: {e.message}") send_email(f"OCI {name} Monitor Error", str(e.message), logger) sys.exit(1) except Exception as e: attempts += 1 error_msg = str(e) logger.error(f"Unexpected error: {error_msg}") if "too many requests" not in error_msg.lower(): # For non-rate-limit unexpected errors, we might want to know, but keep retrying pass logger.info(f"Retrying in {interval}s...") # Weekly Report now = time.time() if now - last_report >= weekly_report_interval: uptime_days = int((now - start_time) / 86400) report_body = ( f"Status: Still running, no capacity available.\n" f"Uptime: {uptime_days} days\n" f"Attempts since last report: {attempts}\n" f"Shape: {shape}" ) send_email(f"OCI {name} Monitor - Weekly Report", report_body, logger) logger.info("Weekly report sent.") last_report = now attempts = 0 time.sleep(interval)