Gan, Jimmy
119 lines
3.9 KiB
Python
119 lines
3.9 KiB
Python
import oci
|
|
import time
|
|
import smtplib
|
|
import logging
|
|
import sys
|
|
import os
|
|
from email.mime.text import MIMEText
|
|
from dotenv import load_dotenv
|
|
|
|
# Load environment variables
|
|
load_dotenv(os.path.join(os.path.dirname(__file__), ".env"))
|
|
|
|
# Constants from environment
|
|
EMAIL_TO = os.environ.get("EMAIL_TO")
|
|
EMAIL_FROM = os.environ.get("EMAIL_FROM")
|
|
EMAIL_APP_PASSWORD = os.environ.get("EMAIL_APP_PASSWORD")
|
|
|
|
def get_logger(name, log_file=None):
|
|
logger = logging.getLogger(name)
|
|
logger.setLevel(logging.INFO)
|
|
|
|
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
|
|
|
# Console handler
|
|
ch = logging.StreamHandler()
|
|
ch.setFormatter(formatter)
|
|
logger.addHandler(ch)
|
|
|
|
# File handler
|
|
if log_file:
|
|
fh = logging.FileHandler(log_file)
|
|
fh.setFormatter(formatter)
|
|
logger.addHandler(fh)
|
|
|
|
return logger
|
|
|
|
def send_email(subject, body, logger=None):
|
|
if not all([EMAIL_FROM, EMAIL_TO, EMAIL_APP_PASSWORD]):
|
|
if logger:
|
|
logger.error("Email configuration missing. Cannot send email.")
|
|
return
|
|
|
|
msg = MIMEText(body)
|
|
msg["Subject"] = subject
|
|
msg["From"] = EMAIL_FROM
|
|
msg["To"] = EMAIL_TO
|
|
|
|
try:
|
|
with smtplib.SMTP_SSL("smtp.gmail.com", 465) as s:
|
|
s.login(EMAIL_FROM, EMAIL_APP_PASSWORD)
|
|
s.send_message(msg)
|
|
except Exception as e:
|
|
if logger:
|
|
logger.error(f"Failed to send email: {e}")
|
|
|
|
def run_monitor(name, shape, launch_details_func, interval=60, weekly_report_interval=7*24*3600):
|
|
logger = get_logger(name, f"{name.lower()}.log")
|
|
logger.info(f"Starting {name} instance monitor ({shape})...")
|
|
|
|
config = oci.config.from_file()
|
|
compute = oci.core.ComputeClient(config)
|
|
|
|
start_time = time.time()
|
|
last_report = start_time
|
|
attempts = 0
|
|
|
|
while True:
|
|
try:
|
|
launch_details = launch_details_func()
|
|
resp = compute.launch_instance(launch_details)
|
|
|
|
instance_id = resp.data.id
|
|
msg = f"{name} instance created!\nID: {instance_id}"
|
|
logger.info(msg)
|
|
send_email(f"OCI {name} Instance Created!", msg, logger)
|
|
sys.exit(0)
|
|
|
|
except oci.exceptions.ServiceError as e:
|
|
msg_lower = str(e.message).lower()
|
|
attempts += 1
|
|
|
|
if "capacity" in msg_lower or "host" in msg_lower or e.status >= 500:
|
|
reason = "No capacity" if "capacity" in msg_lower else e.message
|
|
logger.info(f"{reason}. Retrying in {interval}s...")
|
|
elif "too many requests" in msg_lower or e.status == 429:
|
|
logger.warning(f"Rate limited. Retrying in {interval}s...")
|
|
elif "limit" in msg_lower:
|
|
logger.warning(f"Limit reached: {e.message}. Retrying in {interval}s...")
|
|
else:
|
|
logger.error(f"Unexpected OCI error: {e.message}")
|
|
send_email(f"OCI {name} Monitor Error", str(e.message), logger)
|
|
sys.exit(1)
|
|
|
|
except Exception as e:
|
|
attempts += 1
|
|
error_msg = str(e)
|
|
logger.error(f"Unexpected error: {error_msg}")
|
|
if "too many requests" not in error_msg.lower():
|
|
# For non-rate-limit unexpected errors, we might want to know, but keep retrying
|
|
pass
|
|
logger.info(f"Retrying in {interval}s...")
|
|
|
|
# Weekly Report
|
|
now = time.time()
|
|
if now - last_report >= weekly_report_interval:
|
|
uptime_days = int((now - start_time) / 86400)
|
|
report_body = (
|
|
f"Status: Still running, no capacity available.\n"
|
|
f"Uptime: {uptime_days} days\n"
|
|
f"Attempts since last report: {attempts}\n"
|
|
f"Shape: {shape}"
|
|
)
|
|
send_email(f"OCI {name} Monitor - Weekly Report", report_body, logger)
|
|
logger.info("Weekly report sent.")
|
|
last_report = now
|
|
attempts = 0
|
|
|
|
time.sleep(interval)
|