Refactor ARM monitor to use oci_common module
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+17
-65
@@ -1,88 +1,40 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
import oci
|
import oci
|
||||||
import time
|
|
||||||
import smtplib
|
|
||||||
import sys
|
|
||||||
from email.mime.text import MIMEText
|
|
||||||
|
|
||||||
import os
|
import os
|
||||||
from dotenv import load_dotenv
|
from oci_common import run_monitor
|
||||||
load_dotenv(os.path.join(os.path.dirname(__file__), ".env"))
|
|
||||||
|
|
||||||
|
# Load configuration from environment
|
||||||
COMPARTMENT_ID = os.environ["COMPARTMENT_ID"]
|
COMPARTMENT_ID = os.environ["COMPARTMENT_ID"]
|
||||||
AVAILABILITY_DOMAIN = os.environ["AVAILABILITY_DOMAIN"]
|
AVAILABILITY_DOMAIN = os.environ["AVAILABILITY_DOMAIN"]
|
||||||
SUBNET_ID = os.environ["SUBNET_ID"]
|
SUBNET_ID = os.environ["SUBNET_ID"]
|
||||||
IMAGE_ID = os.environ["IMAGE_ID"]
|
IMAGE_ID = os.environ["IMAGE_ID"]
|
||||||
SSH_KEY = os.environ["SSH_KEY"]
|
SSH_KEY = os.environ["SSH_KEY"]
|
||||||
DISPLAY_NAME = os.environ.get("DISPLAY_NAME", "free-arm-instance")
|
DISPLAY_NAME = os.environ.get("DISPLAY_NAME", "free-arm-instance")
|
||||||
|
|
||||||
SHAPE = "VM.Standard.A1.Flex"
|
SHAPE = "VM.Standard.A1.Flex"
|
||||||
OCPUS = 4
|
OCPUS = 4
|
||||||
MEMORY_GB = 24
|
MEMORY_GB = 24
|
||||||
INTERVAL = 60
|
INTERVAL = 60 # Check every minute
|
||||||
WEEKLY_REPORT_INTERVAL = 7 * 24 * 3600
|
|
||||||
|
|
||||||
EMAIL_TO = os.environ["EMAIL_TO"]
|
def get_launch_details():
|
||||||
EMAIL_FROM = os.environ["EMAIL_FROM"]
|
return oci.core.models.LaunchInstanceDetails(
|
||||||
EMAIL_APP_PASSWORD = os.environ["EMAIL_APP_PASSWORD"]
|
|
||||||
|
|
||||||
def send_email(subject, body):
|
|
||||||
msg = MIMEText(body)
|
|
||||||
msg["Subject"] = subject
|
|
||||||
msg["From"] = EMAIL_FROM
|
|
||||||
msg["To"] = EMAIL_TO
|
|
||||||
with smtplib.SMTP_SSL("smtp.gmail.com", 465) as s:
|
|
||||||
s.login(EMAIL_FROM, EMAIL_APP_PASSWORD)
|
|
||||||
s.send_message(msg)
|
|
||||||
|
|
||||||
config = oci.config.from_file()
|
|
||||||
compute = oci.core.ComputeClient(config)
|
|
||||||
|
|
||||||
print("Starting ARM instance monitor...", flush=True)
|
|
||||||
start_time = time.time()
|
|
||||||
last_report = start_time
|
|
||||||
attempts = 0
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
resp = compute.launch_instance(
|
|
||||||
oci.core.models.LaunchInstanceDetails(
|
|
||||||
compartment_id=COMPARTMENT_ID,
|
compartment_id=COMPARTMENT_ID,
|
||||||
availability_domain=AVAILABILITY_DOMAIN,
|
availability_domain=AVAILABILITY_DOMAIN,
|
||||||
shape=SHAPE,
|
shape=SHAPE,
|
||||||
shape_config=oci.core.models.LaunchInstanceShapeConfigDetails(ocpus=OCPUS, memory_in_gbs=MEMORY_GB),
|
shape_config=oci.core.models.LaunchInstanceShapeConfigDetails(
|
||||||
|
ocpus=OCPUS,
|
||||||
|
memory_in_gbs=MEMORY_GB
|
||||||
|
),
|
||||||
image_id=IMAGE_ID,
|
image_id=IMAGE_ID,
|
||||||
subnet_id=SUBNET_ID,
|
subnet_id=SUBNET_ID,
|
||||||
display_name=DISPLAY_NAME,
|
display_name=DISPLAY_NAME,
|
||||||
metadata={"ssh_authorized_keys": SSH_KEY},
|
metadata={"ssh_authorized_keys": SSH_KEY},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
run_monitor(
|
||||||
|
name="ARM",
|
||||||
|
shape=f"{SHAPE} ({OCPUS} OCPUs, {MEMORY_GB}GB RAM)",
|
||||||
|
launch_details_func=get_launch_details,
|
||||||
|
interval=INTERVAL
|
||||||
)
|
)
|
||||||
instance_id = resp.data.id
|
|
||||||
msg = f"ARM instance created!\nID: {instance_id}"
|
|
||||||
print(msg, flush=True)
|
|
||||||
send_email("OCI ARM Instance Created!", msg)
|
|
||||||
sys.exit(0)
|
|
||||||
except oci.exceptions.ServiceError as e:
|
|
||||||
msg_lower = str(e.message).lower()
|
|
||||||
if "capacity" in msg_lower or "host" in msg_lower or e.status >= 500:
|
|
||||||
attempts += 1
|
|
||||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time() + 8*3600))} {'No capacity' if 'capacity' in msg_lower else e.message}. Retrying in {INTERVAL}s...", flush=True)
|
|
||||||
else:
|
|
||||||
print(f"Unexpected error: {e.message}", flush=True)
|
|
||||||
send_email("OCI ARM Monitor Error", str(e.message))
|
|
||||||
sys.exit(1)
|
|
||||||
except Exception as e:
|
|
||||||
attempts += 1
|
|
||||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time() + 8*3600))} {e}. Retrying in {INTERVAL}s...", flush=True)
|
|
||||||
now = time.time()
|
|
||||||
if now - last_report >= WEEKLY_REPORT_INTERVAL:
|
|
||||||
uptime_days = int((now - start_time) / 86400)
|
|
||||||
send_email(
|
|
||||||
"OCI ARM Monitor - Weekly Report",
|
|
||||||
f"Status: Still running, no capacity available.\n"
|
|
||||||
f"Uptime: {uptime_days} days\n"
|
|
||||||
f"Attempts since last report: {attempts}\n"
|
|
||||||
f"Shape: {SHAPE} ({OCPUS} OCPUs, {MEMORY_GB}GB RAM)",
|
|
||||||
)
|
|
||||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(now + 8*3600))} Weekly report sent.", flush=True)
|
|
||||||
last_report = now
|
|
||||||
attempts = 0
|
|
||||||
time.sleep(INTERVAL)
|
|
||||||
|
|||||||
+118
@@ -0,0 +1,118 @@
|
|||||||
|
import oci
|
||||||
|
import time
|
||||||
|
import smtplib
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
from email.mime.text import MIMEText
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
# Load environment variables
|
||||||
|
load_dotenv(os.path.join(os.path.dirname(__file__), ".env"))
|
||||||
|
|
||||||
|
# Constants from environment
|
||||||
|
EMAIL_TO = os.environ.get("EMAIL_TO")
|
||||||
|
EMAIL_FROM = os.environ.get("EMAIL_FROM")
|
||||||
|
EMAIL_APP_PASSWORD = os.environ.get("EMAIL_APP_PASSWORD")
|
||||||
|
|
||||||
|
def get_logger(name, log_file=None):
|
||||||
|
logger = logging.getLogger(name)
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
||||||
|
|
||||||
|
# Console handler
|
||||||
|
ch = logging.StreamHandler()
|
||||||
|
ch.setFormatter(formatter)
|
||||||
|
logger.addHandler(ch)
|
||||||
|
|
||||||
|
# File handler
|
||||||
|
if log_file:
|
||||||
|
fh = logging.FileHandler(log_file)
|
||||||
|
fh.setFormatter(formatter)
|
||||||
|
logger.addHandler(fh)
|
||||||
|
|
||||||
|
return logger
|
||||||
|
|
||||||
|
def send_email(subject, body, logger=None):
|
||||||
|
if not all([EMAIL_FROM, EMAIL_TO, EMAIL_APP_PASSWORD]):
|
||||||
|
if logger:
|
||||||
|
logger.error("Email configuration missing. Cannot send email.")
|
||||||
|
return
|
||||||
|
|
||||||
|
msg = MIMEText(body)
|
||||||
|
msg["Subject"] = subject
|
||||||
|
msg["From"] = EMAIL_FROM
|
||||||
|
msg["To"] = EMAIL_TO
|
||||||
|
|
||||||
|
try:
|
||||||
|
with smtplib.SMTP_SSL("smtp.gmail.com", 465) as s:
|
||||||
|
s.login(EMAIL_FROM, EMAIL_APP_PASSWORD)
|
||||||
|
s.send_message(msg)
|
||||||
|
except Exception as e:
|
||||||
|
if logger:
|
||||||
|
logger.error(f"Failed to send email: {e}")
|
||||||
|
|
||||||
|
def run_monitor(name, shape, launch_details_func, interval=60, weekly_report_interval=7*24*3600):
|
||||||
|
logger = get_logger(name, f"{name.lower()}.log")
|
||||||
|
logger.info(f"Starting {name} instance monitor ({shape})...")
|
||||||
|
|
||||||
|
config = oci.config.from_file()
|
||||||
|
compute = oci.core.ComputeClient(config)
|
||||||
|
|
||||||
|
start_time = time.time()
|
||||||
|
last_report = start_time
|
||||||
|
attempts = 0
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
launch_details = launch_details_func()
|
||||||
|
resp = compute.launch_instance(launch_details)
|
||||||
|
|
||||||
|
instance_id = resp.data.id
|
||||||
|
msg = f"{name} instance created!\nID: {instance_id}"
|
||||||
|
logger.info(msg)
|
||||||
|
send_email(f"OCI {name} Instance Created!", msg, logger)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
except oci.exceptions.ServiceError as e:
|
||||||
|
msg_lower = str(e.message).lower()
|
||||||
|
attempts += 1
|
||||||
|
|
||||||
|
if "capacity" in msg_lower or "host" in msg_lower or e.status >= 500:
|
||||||
|
reason = "No capacity" if "capacity" in msg_lower else e.message
|
||||||
|
logger.info(f"{reason}. Retrying in {interval}s...")
|
||||||
|
elif "too many requests" in msg_lower or e.status == 429:
|
||||||
|
logger.warning(f"Rate limited. Retrying in {interval}s...")
|
||||||
|
elif "limit" in msg_lower:
|
||||||
|
logger.warning(f"Limit reached: {e.message}. Retrying in {interval}s...")
|
||||||
|
else:
|
||||||
|
logger.error(f"Unexpected OCI error: {e.message}")
|
||||||
|
send_email(f"OCI {name} Monitor Error", str(e.message), logger)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
attempts += 1
|
||||||
|
error_msg = str(e)
|
||||||
|
logger.error(f"Unexpected error: {error_msg}")
|
||||||
|
if "too many requests" not in error_msg.lower():
|
||||||
|
# For non-rate-limit unexpected errors, we might want to know, but keep retrying
|
||||||
|
pass
|
||||||
|
logger.info(f"Retrying in {interval}s...")
|
||||||
|
|
||||||
|
# Weekly Report
|
||||||
|
now = time.time()
|
||||||
|
if now - last_report >= weekly_report_interval:
|
||||||
|
uptime_days = int((now - start_time) / 86400)
|
||||||
|
report_body = (
|
||||||
|
f"Status: Still running, no capacity available.\n"
|
||||||
|
f"Uptime: {uptime_days} days\n"
|
||||||
|
f"Attempts since last report: {attempts}\n"
|
||||||
|
f"Shape: {shape}"
|
||||||
|
)
|
||||||
|
send_email(f"OCI {name} Monitor - Weekly Report", report_body, logger)
|
||||||
|
logger.info("Weekly report sent.")
|
||||||
|
last_report = now
|
||||||
|
attempts = 0
|
||||||
|
|
||||||
|
time.sleep(interval)
|
||||||
Reference in New Issue
Block a user