diff --git a/dashboard/backend/main.py b/dashboard/backend/main.py index bf1f965..a4c34d4 100644 --- a/dashboard/backend/main.py +++ b/dashboard/backend/main.py @@ -176,9 +176,25 @@ async def audit_log(request: Request, call_next): async def monitor_containers(): import docker + import time + import logging from config import DOCKER_HOST + _monitor_logger = logging.getLogger(__name__) + + def _docker_retry(fn, max_retries=3, base_delay=1): + for attempt in range(max_retries): + try: + return fn() + except Exception as e: + delay = base_delay * (2 ** attempt) + _monitor_logger.warning("Docker monitor error (attempt %d/%d): %s. Retrying in %ds...", attempt+1, max_retries, e, delay) + if attempt < max_retries - 1: + time.sleep(delay) + _monitor_logger.error("Docker monitor failed after %d attempts", max_retries) + return None + client = docker.DockerClient(base_url=DOCKER_HOST) # Store initial states @@ -186,7 +202,9 @@ async def monitor_containers(): while True: try: - containers = await asyncio.to_thread(client.containers.list, all=True) + containers = await asyncio.to_thread( + lambda: docker_retry(lambda: client.containers.list(all=True)) or [] + ) for c in containers: current_status = c.status if c.id in previous_states: diff --git a/dashboard/backend/routers/docker_router.py b/dashboard/backend/routers/docker_router.py index 76177a1..69f5653 100644 --- a/dashboard/backend/routers/docker_router.py +++ b/dashboard/backend/routers/docker_router.py @@ -4,6 +4,27 @@ from fastapi import APIRouter, Depends, HTTPException, Query from config import DOCKER_HOST from rbac import require_admin + +import time +import logging + +logger = logging.getLogger(__name__) + +def docker_retry(fn, max_retries=3, base_delay=1): + """Execute fn with exponential backoff. Returns (result, True) or (None, False).""" + for attempt in range(max_retries): + try: + result = fn() + return result + except Exception as e: + delay = base_delay * (2 ** attempt) + logger.warning("Docker API error (attempt %d/%d): %s. Retrying in %ds...", attempt+1, max_retries, e, delay) + if attempt < max_retries - 1: + time.sleep(delay) + logger.error("Docker API failed after %d attempts", max_retries) + return None + + router = APIRouter() _client = None @@ -20,6 +41,9 @@ def get_docker_client(): @router.get("/containers") def list_containers(): client = get_docker_client() + result = docker_retry(lambda: client.containers.list(all=True)) + if result is None: + return [] return [ { "id": c.short_id, @@ -29,7 +53,7 @@ def list_containers(): "image": c.image.tags[0] if c.image.tags else str(c.image.id)[:20], "ports": c.ports, } - for c in client.containers.list(all=True) + for c in result ] @@ -38,13 +62,22 @@ def container_action(container_id: str, action: str): if action not in ("start", "stop", "restart"): raise HTTPException(status_code=400, detail="Invalid action. Must be one of: start, stop, restart") client = get_docker_client() - c = client.containers.get(container_id) - getattr(c, action)() + def _do(): + c = client.containers.get(container_id) + getattr(c, action)() + return True + if docker_retry(_do) is None: + raise HTTPException(status_code=503, detail="Docker API unavailable") return {"ok": True} @router.get("/containers/{container_id}/logs") def container_logs(container_id: str, tail: int = Query(200, le=10000)): client = get_docker_client() - c = client.containers.get(container_id) - return {"logs": c.logs(tail=tail, timestamps=True).decode(errors="replace")} + def _do(): + c = client.containers.get(container_id) + return {"logs": c.logs(tail=tail, timestamps=True).decode(errors="replace")} + result = docker_retry(_do) + if result is None: + return {"logs": ""} + return result