fix: S06.3 add retry/backoff to Docker API calls (3 retries, exponential backoff 1s/2s/4s)

This commit is contained in:
Gan, Jimmy
2026-07-11 01:35:35 +08:00
parent a85bbd2411
commit d325c84457
2 changed files with 57 additions and 6 deletions
+19 -1
View File
@@ -176,9 +176,25 @@ async def audit_log(request: Request, call_next):
async def monitor_containers():
import docker
import time
import logging
from config import DOCKER_HOST
_monitor_logger = logging.getLogger(__name__)
def _docker_retry(fn, max_retries=3, base_delay=1):
for attempt in range(max_retries):
try:
return fn()
except Exception as e:
delay = base_delay * (2 ** attempt)
_monitor_logger.warning("Docker monitor error (attempt %d/%d): %s. Retrying in %ds...", attempt+1, max_retries, e, delay)
if attempt < max_retries - 1:
time.sleep(delay)
_monitor_logger.error("Docker monitor failed after %d attempts", max_retries)
return None
client = docker.DockerClient(base_url=DOCKER_HOST)
# Store initial states
@@ -186,7 +202,9 @@ async def monitor_containers():
while True:
try:
containers = await asyncio.to_thread(client.containers.list, all=True)
containers = await asyncio.to_thread(
lambda: docker_retry(lambda: client.containers.list(all=True)) or []
)
for c in containers:
current_status = c.status
if c.id in previous_states: