fix: S06.3 add retry/backoff to Docker API calls (3 retries, exponential backoff 1s/2s/4s)

dashboard/backend/main.py

@@ -176,9 +176,25 @@ async def audit_log(request: Request, call_next):
 
 async def monitor_containers():
     import docker
+    import time
+    import logging
 
     from config import DOCKER_HOST
 
+    _monitor_logger = logging.getLogger(__name__)
+
+    def _docker_retry(fn, max_retries=3, base_delay=1):
+        for attempt in range(max_retries):
+            try:
+                return fn()
+            except Exception as e:
+                delay = base_delay * (2 ** attempt)
+                _monitor_logger.warning("Docker monitor error (attempt %d/%d): %s. Retrying in %ds...", attempt+1, max_retries, e, delay)
+                if attempt < max_retries - 1:
+                    time.sleep(delay)
+        _monitor_logger.error("Docker monitor failed after %d attempts", max_retries)
+        return None
+
     client = docker.DockerClient(base_url=DOCKER_HOST)
 
     # Store initial states
@@ -186,7 +202,9 @@ async def monitor_containers():
 
     while True:
         try:
-            containers = await asyncio.to_thread(client.containers.list, all=True)
+            containers = await asyncio.to_thread(
+                lambda: docker_retry(lambda: client.containers.list(all=True)) or []
+            )
             for c in containers:
                 current_status = c.status
                 if c.id in previous_states:

dashboard/backend/routers/docker_router.py

@@ -4,6 +4,27 @@ from fastapi import APIRouter, Depends, HTTPException, Query
 from config import DOCKER_HOST
 from rbac import require_admin
 
+
+import time
+import logging
+
+logger = logging.getLogger(__name__)
+
+def docker_retry(fn, max_retries=3, base_delay=1):
+    """Execute fn with exponential backoff. Returns (result, True) or (None, False)."""
+    for attempt in range(max_retries):
+        try:
+            result = fn()
+            return result
+        except Exception as e:
+            delay = base_delay * (2 ** attempt)
+            logger.warning("Docker API error (attempt %d/%d): %s. Retrying in %ds...", attempt+1, max_retries, e, delay)
+            if attempt < max_retries - 1:
+                time.sleep(delay)
+    logger.error("Docker API failed after %d attempts", max_retries)
+    return None
+
+
 router = APIRouter()
 
 _client = None
@@ -20,6 +41,9 @@ def get_docker_client():
 @router.get("/containers")
 def list_containers():
     client = get_docker_client()
+    result = docker_retry(lambda: client.containers.list(all=True))
+    if result is None:
+        return []
     return [
         {
             "id": c.short_id,
@@ -29,7 +53,7 @@ def list_containers():
             "image": c.image.tags[0] if c.image.tags else str(c.image.id)[:20],
             "ports": c.ports,
         }
-        for c in client.containers.list(all=True)
+        for c in result
     ]
 
 
@@ -38,13 +62,22 @@ def container_action(container_id: str, action: str):
     if action not in ("start", "stop", "restart"):
         raise HTTPException(status_code=400, detail="Invalid action. Must be one of: start, stop, restart")
     client = get_docker_client()
+    def _do():
         c = client.containers.get(container_id)
         getattr(c, action)()
+        return True
+    if docker_retry(_do) is None:
+        raise HTTPException(status_code=503, detail="Docker API unavailable")
     return {"ok": True}
 
 
 @router.get("/containers/{container_id}/logs")
 def container_logs(container_id: str, tail: int = Query(200, le=10000)):
     client = get_docker_client()
+    def _do():
         c = client.containers.get(container_id)
         return {"logs": c.logs(tail=tail, timestamps=True).decode(errors="replace")}
+    result = docker_retry(_do)
+    if result is None:
+        return {"logs": ""}
+    return result