Refactor ARM monitor to use oci_common module
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Gan, Jimmy
+17
-65
@@ -1,88 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
import oci
|
||||
import time
|
||||
import smtplib
|
||||
import sys
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv(os.path.join(os.path.dirname(__file__), ".env"))
|
||||
from oci_common import run_monitor
|
||||
|
||||
# Load configuration from environment
|
||||
COMPARTMENT_ID = os.environ["COMPARTMENT_ID"]
|
||||
AVAILABILITY_DOMAIN = os.environ["AVAILABILITY_DOMAIN"]
|
||||
SUBNET_ID = os.environ["SUBNET_ID"]
|
||||
IMAGE_ID = os.environ["IMAGE_ID"]
|
||||
SSH_KEY = os.environ["SSH_KEY"]
|
||||
DISPLAY_NAME = os.environ.get("DISPLAY_NAME", "free-arm-instance")
|
||||
|
||||
SHAPE = "VM.Standard.A1.Flex"
|
||||
OCPUS = 4
|
||||
MEMORY_GB = 24
|
||||
INTERVAL = 60
|
||||
WEEKLY_REPORT_INTERVAL = 7 * 24 * 3600
|
||||
INTERVAL = 60 # Check every minute
|
||||
|
||||
EMAIL_TO = os.environ["EMAIL_TO"]
|
||||
EMAIL_FROM = os.environ["EMAIL_FROM"]
|
||||
EMAIL_APP_PASSWORD = os.environ["EMAIL_APP_PASSWORD"]
|
||||
|
||||
def send_email(subject, body):
|
||||
msg = MIMEText(body)
|
||||
msg["Subject"] = subject
|
||||
msg["From"] = EMAIL_FROM
|
||||
msg["To"] = EMAIL_TO
|
||||
with smtplib.SMTP_SSL("smtp.gmail.com", 465) as s:
|
||||
s.login(EMAIL_FROM, EMAIL_APP_PASSWORD)
|
||||
s.send_message(msg)
|
||||
|
||||
config = oci.config.from_file()
|
||||
compute = oci.core.ComputeClient(config)
|
||||
|
||||
print("Starting ARM instance monitor...", flush=True)
|
||||
start_time = time.time()
|
||||
last_report = start_time
|
||||
attempts = 0
|
||||
while True:
|
||||
try:
|
||||
resp = compute.launch_instance(
|
||||
oci.core.models.LaunchInstanceDetails(
|
||||
def get_launch_details():
|
||||
return oci.core.models.LaunchInstanceDetails(
|
||||
compartment_id=COMPARTMENT_ID,
|
||||
availability_domain=AVAILABILITY_DOMAIN,
|
||||
shape=SHAPE,
|
||||
shape_config=oci.core.models.LaunchInstanceShapeConfigDetails(ocpus=OCPUS, memory_in_gbs=MEMORY_GB),
|
||||
shape_config=oci.core.models.LaunchInstanceShapeConfigDetails(
|
||||
ocpus=OCPUS,
|
||||
memory_in_gbs=MEMORY_GB
|
||||
),
|
||||
image_id=IMAGE_ID,
|
||||
subnet_id=SUBNET_ID,
|
||||
display_name=DISPLAY_NAME,
|
||||
metadata={"ssh_authorized_keys": SSH_KEY},
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_monitor(
|
||||
name="ARM",
|
||||
shape=f"{SHAPE} ({OCPUS} OCPUs, {MEMORY_GB}GB RAM)",
|
||||
launch_details_func=get_launch_details,
|
||||
interval=INTERVAL
|
||||
)
|
||||
instance_id = resp.data.id
|
||||
msg = f"ARM instance created!\nID: {instance_id}"
|
||||
print(msg, flush=True)
|
||||
send_email("OCI ARM Instance Created!", msg)
|
||||
sys.exit(0)
|
||||
except oci.exceptions.ServiceError as e:
|
||||
msg_lower = str(e.message).lower()
|
||||
if "capacity" in msg_lower or "host" in msg_lower or e.status >= 500:
|
||||
attempts += 1
|
||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time() + 8*3600))} {'No capacity' if 'capacity' in msg_lower else e.message}. Retrying in {INTERVAL}s...", flush=True)
|
||||
else:
|
||||
print(f"Unexpected error: {e.message}", flush=True)
|
||||
send_email("OCI ARM Monitor Error", str(e.message))
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
attempts += 1
|
||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time() + 8*3600))} {e}. Retrying in {INTERVAL}s...", flush=True)
|
||||
now = time.time()
|
||||
if now - last_report >= WEEKLY_REPORT_INTERVAL:
|
||||
uptime_days = int((now - start_time) / 86400)
|
||||
send_email(
|
||||
"OCI ARM Monitor - Weekly Report",
|
||||
f"Status: Still running, no capacity available.\n"
|
||||
f"Uptime: {uptime_days} days\n"
|
||||
f"Attempts since last report: {attempts}\n"
|
||||
f"Shape: {SHAPE} ({OCPUS} OCPUs, {MEMORY_GB}GB RAM)",
|
||||
)
|
||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(now + 8*3600))} Weekly report sent.", flush=True)
|
||||
last_report = now
|
||||
attempts = 0
|
||||
time.sleep(INTERVAL)
|
||||
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
import oci
|
||||
import time
|
||||
import smtplib
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
from email.mime.text import MIMEText
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables
|
||||
load_dotenv(os.path.join(os.path.dirname(__file__), ".env"))
|
||||
|
||||
# Constants from environment
|
||||
EMAIL_TO = os.environ.get("EMAIL_TO")
|
||||
EMAIL_FROM = os.environ.get("EMAIL_FROM")
|
||||
EMAIL_APP_PASSWORD = os.environ.get("EMAIL_APP_PASSWORD")
|
||||
|
||||
def get_logger(name, log_file=None):
|
||||
logger = logging.getLogger(name)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
||||
|
||||
# Console handler
|
||||
ch = logging.StreamHandler()
|
||||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
# File handler
|
||||
if log_file:
|
||||
fh = logging.FileHandler(log_file)
|
||||
fh.setFormatter(formatter)
|
||||
logger.addHandler(fh)
|
||||
|
||||
return logger
|
||||
|
||||
def send_email(subject, body, logger=None):
|
||||
if not all([EMAIL_FROM, EMAIL_TO, EMAIL_APP_PASSWORD]):
|
||||
if logger:
|
||||
logger.error("Email configuration missing. Cannot send email.")
|
||||
return
|
||||
|
||||
msg = MIMEText(body)
|
||||
msg["Subject"] = subject
|
||||
msg["From"] = EMAIL_FROM
|
||||
msg["To"] = EMAIL_TO
|
||||
|
||||
try:
|
||||
with smtplib.SMTP_SSL("smtp.gmail.com", 465) as s:
|
||||
s.login(EMAIL_FROM, EMAIL_APP_PASSWORD)
|
||||
s.send_message(msg)
|
||||
except Exception as e:
|
||||
if logger:
|
||||
logger.error(f"Failed to send email: {e}")
|
||||
|
||||
def run_monitor(name, shape, launch_details_func, interval=60, weekly_report_interval=7*24*3600):
|
||||
logger = get_logger(name, f"{name.lower()}.log")
|
||||
logger.info(f"Starting {name} instance monitor ({shape})...")
|
||||
|
||||
config = oci.config.from_file()
|
||||
compute = oci.core.ComputeClient(config)
|
||||
|
||||
start_time = time.time()
|
||||
last_report = start_time
|
||||
attempts = 0
|
||||
|
||||
while True:
|
||||
try:
|
||||
launch_details = launch_details_func()
|
||||
resp = compute.launch_instance(launch_details)
|
||||
|
||||
instance_id = resp.data.id
|
||||
msg = f"{name} instance created!\nID: {instance_id}"
|
||||
logger.info(msg)
|
||||
send_email(f"OCI {name} Instance Created!", msg, logger)
|
||||
sys.exit(0)
|
||||
|
||||
except oci.exceptions.ServiceError as e:
|
||||
msg_lower = str(e.message).lower()
|
||||
attempts += 1
|
||||
|
||||
if "capacity" in msg_lower or "host" in msg_lower or e.status >= 500:
|
||||
reason = "No capacity" if "capacity" in msg_lower else e.message
|
||||
logger.info(f"{reason}. Retrying in {interval}s...")
|
||||
elif "too many requests" in msg_lower or e.status == 429:
|
||||
logger.warning(f"Rate limited. Retrying in {interval}s...")
|
||||
elif "limit" in msg_lower:
|
||||
logger.warning(f"Limit reached: {e.message}. Retrying in {interval}s...")
|
||||
else:
|
||||
logger.error(f"Unexpected OCI error: {e.message}")
|
||||
send_email(f"OCI {name} Monitor Error", str(e.message), logger)
|
||||
sys.exit(1)
|
||||
|
||||
except Exception as e:
|
||||
attempts += 1
|
||||
error_msg = str(e)
|
||||
logger.error(f"Unexpected error: {error_msg}")
|
||||
if "too many requests" not in error_msg.lower():
|
||||
# For non-rate-limit unexpected errors, we might want to know, but keep retrying
|
||||
pass
|
||||
logger.info(f"Retrying in {interval}s...")
|
||||
|
||||
# Weekly Report
|
||||
now = time.time()
|
||||
if now - last_report >= weekly_report_interval:
|
||||
uptime_days = int((now - start_time) / 86400)
|
||||
report_body = (
|
||||
f"Status: Still running, no capacity available.\n"
|
||||
f"Uptime: {uptime_days} days\n"
|
||||
f"Attempts since last report: {attempts}\n"
|
||||
f"Shape: {shape}"
|
||||
)
|
||||
send_email(f"OCI {name} Monitor - Weekly Report", report_body, logger)
|
||||
logger.info("Weekly report sent.")
|
||||
last_report = now
|
||||
attempts = 0
|
||||
|
||||
time.sleep(interval)
|
||||
Reference in New Issue
Block a user