diff --git a/oci-arm-monitor.py b/oci-arm-monitor.py index 4f0f548..c047437 100644 --- a/oci-arm-monitor.py +++ b/oci-arm-monitor.py @@ -1,88 +1,40 @@ #!/usr/bin/env python3 import oci -import time -import smtplib -import sys -from email.mime.text import MIMEText - import os -from dotenv import load_dotenv -load_dotenv(os.path.join(os.path.dirname(__file__), ".env")) +from oci_common import run_monitor +# Load configuration from environment COMPARTMENT_ID = os.environ["COMPARTMENT_ID"] AVAILABILITY_DOMAIN = os.environ["AVAILABILITY_DOMAIN"] SUBNET_ID = os.environ["SUBNET_ID"] IMAGE_ID = os.environ["IMAGE_ID"] SSH_KEY = os.environ["SSH_KEY"] DISPLAY_NAME = os.environ.get("DISPLAY_NAME", "free-arm-instance") + SHAPE = "VM.Standard.A1.Flex" OCPUS = 4 MEMORY_GB = 24 -INTERVAL = 60 -WEEKLY_REPORT_INTERVAL = 7 * 24 * 3600 +INTERVAL = 60 # Check every minute -EMAIL_TO = os.environ["EMAIL_TO"] -EMAIL_FROM = os.environ["EMAIL_FROM"] -EMAIL_APP_PASSWORD = os.environ["EMAIL_APP_PASSWORD"] +def get_launch_details(): + return oci.core.models.LaunchInstanceDetails( + compartment_id=COMPARTMENT_ID, + availability_domain=AVAILABILITY_DOMAIN, + shape=SHAPE, + shape_config=oci.core.models.LaunchInstanceShapeConfigDetails( + ocpus=OCPUS, + memory_in_gbs=MEMORY_GB + ), + image_id=IMAGE_ID, + subnet_id=SUBNET_ID, + display_name=DISPLAY_NAME, + metadata={"ssh_authorized_keys": SSH_KEY}, + ) -def send_email(subject, body): - msg = MIMEText(body) - msg["Subject"] = subject - msg["From"] = EMAIL_FROM - msg["To"] = EMAIL_TO - with smtplib.SMTP_SSL("smtp.gmail.com", 465) as s: - s.login(EMAIL_FROM, EMAIL_APP_PASSWORD) - s.send_message(msg) - -config = oci.config.from_file() -compute = oci.core.ComputeClient(config) - -print("Starting ARM instance monitor...", flush=True) -start_time = time.time() -last_report = start_time -attempts = 0 -while True: - try: - resp = compute.launch_instance( - oci.core.models.LaunchInstanceDetails( - compartment_id=COMPARTMENT_ID, - availability_domain=AVAILABILITY_DOMAIN, - shape=SHAPE, - shape_config=oci.core.models.LaunchInstanceShapeConfigDetails(ocpus=OCPUS, memory_in_gbs=MEMORY_GB), - image_id=IMAGE_ID, - subnet_id=SUBNET_ID, - display_name=DISPLAY_NAME, - metadata={"ssh_authorized_keys": SSH_KEY}, - ) - ) - instance_id = resp.data.id - msg = f"ARM instance created!\nID: {instance_id}" - print(msg, flush=True) - send_email("OCI ARM Instance Created!", msg) - sys.exit(0) - except oci.exceptions.ServiceError as e: - msg_lower = str(e.message).lower() - if "capacity" in msg_lower or "host" in msg_lower or e.status >= 500: - attempts += 1 - print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time() + 8*3600))} {'No capacity' if 'capacity' in msg_lower else e.message}. Retrying in {INTERVAL}s...", flush=True) - else: - print(f"Unexpected error: {e.message}", flush=True) - send_email("OCI ARM Monitor Error", str(e.message)) - sys.exit(1) - except Exception as e: - attempts += 1 - print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time() + 8*3600))} {e}. Retrying in {INTERVAL}s...", flush=True) - now = time.time() - if now - last_report >= WEEKLY_REPORT_INTERVAL: - uptime_days = int((now - start_time) / 86400) - send_email( - "OCI ARM Monitor - Weekly Report", - f"Status: Still running, no capacity available.\n" - f"Uptime: {uptime_days} days\n" - f"Attempts since last report: {attempts}\n" - f"Shape: {SHAPE} ({OCPUS} OCPUs, {MEMORY_GB}GB RAM)", - ) - print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(now + 8*3600))} Weekly report sent.", flush=True) - last_report = now - attempts = 0 - time.sleep(INTERVAL) +if __name__ == "__main__": + run_monitor( + name="ARM", + shape=f"{SHAPE} ({OCPUS} OCPUs, {MEMORY_GB}GB RAM)", + launch_details_func=get_launch_details, + interval=INTERVAL + ) diff --git a/oci_common.py b/oci_common.py new file mode 100644 index 0000000..fffb1b7 --- /dev/null +++ b/oci_common.py @@ -0,0 +1,118 @@ +import oci +import time +import smtplib +import logging +import sys +import os +from email.mime.text import MIMEText +from dotenv import load_dotenv + +# Load environment variables +load_dotenv(os.path.join(os.path.dirname(__file__), ".env")) + +# Constants from environment +EMAIL_TO = os.environ.get("EMAIL_TO") +EMAIL_FROM = os.environ.get("EMAIL_FROM") +EMAIL_APP_PASSWORD = os.environ.get("EMAIL_APP_PASSWORD") + +def get_logger(name, log_file=None): + logger = logging.getLogger(name) + logger.setLevel(logging.INFO) + + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + + # Console handler + ch = logging.StreamHandler() + ch.setFormatter(formatter) + logger.addHandler(ch) + + # File handler + if log_file: + fh = logging.FileHandler(log_file) + fh.setFormatter(formatter) + logger.addHandler(fh) + + return logger + +def send_email(subject, body, logger=None): + if not all([EMAIL_FROM, EMAIL_TO, EMAIL_APP_PASSWORD]): + if logger: + logger.error("Email configuration missing. Cannot send email.") + return + + msg = MIMEText(body) + msg["Subject"] = subject + msg["From"] = EMAIL_FROM + msg["To"] = EMAIL_TO + + try: + with smtplib.SMTP_SSL("smtp.gmail.com", 465) as s: + s.login(EMAIL_FROM, EMAIL_APP_PASSWORD) + s.send_message(msg) + except Exception as e: + if logger: + logger.error(f"Failed to send email: {e}") + +def run_monitor(name, shape, launch_details_func, interval=60, weekly_report_interval=7*24*3600): + logger = get_logger(name, f"{name.lower()}.log") + logger.info(f"Starting {name} instance monitor ({shape})...") + + config = oci.config.from_file() + compute = oci.core.ComputeClient(config) + + start_time = time.time() + last_report = start_time + attempts = 0 + + while True: + try: + launch_details = launch_details_func() + resp = compute.launch_instance(launch_details) + + instance_id = resp.data.id + msg = f"{name} instance created!\nID: {instance_id}" + logger.info(msg) + send_email(f"OCI {name} Instance Created!", msg, logger) + sys.exit(0) + + except oci.exceptions.ServiceError as e: + msg_lower = str(e.message).lower() + attempts += 1 + + if "capacity" in msg_lower or "host" in msg_lower or e.status >= 500: + reason = "No capacity" if "capacity" in msg_lower else e.message + logger.info(f"{reason}. Retrying in {interval}s...") + elif "too many requests" in msg_lower or e.status == 429: + logger.warning(f"Rate limited. Retrying in {interval}s...") + elif "limit" in msg_lower: + logger.warning(f"Limit reached: {e.message}. Retrying in {interval}s...") + else: + logger.error(f"Unexpected OCI error: {e.message}") + send_email(f"OCI {name} Monitor Error", str(e.message), logger) + sys.exit(1) + + except Exception as e: + attempts += 1 + error_msg = str(e) + logger.error(f"Unexpected error: {error_msg}") + if "too many requests" not in error_msg.lower(): + # For non-rate-limit unexpected errors, we might want to know, but keep retrying + pass + logger.info(f"Retrying in {interval}s...") + + # Weekly Report + now = time.time() + if now - last_report >= weekly_report_interval: + uptime_days = int((now - start_time) / 86400) + report_body = ( + f"Status: Still running, no capacity available.\n" + f"Uptime: {uptime_days} days\n" + f"Attempts since last report: {attempts}\n" + f"Shape: {shape}" + ) + send_email(f"OCI {name} Monitor - Weekly Report", report_body, logger) + logger.info("Weekly report sent.") + last_report = now + attempts = 0 + + time.sleep(interval)