fix: S06.3 add retry/backoff to Docker API calls (3 retries, exponential backoff 1s/2s/4s)
This commit is contained in:
@@ -176,9 +176,25 @@ async def audit_log(request: Request, call_next):
|
||||
|
||||
async def monitor_containers():
|
||||
import docker
|
||||
import time
|
||||
import logging
|
||||
|
||||
from config import DOCKER_HOST
|
||||
|
||||
_monitor_logger = logging.getLogger(__name__)
|
||||
|
||||
def _docker_retry(fn, max_retries=3, base_delay=1):
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return fn()
|
||||
except Exception as e:
|
||||
delay = base_delay * (2 ** attempt)
|
||||
_monitor_logger.warning("Docker monitor error (attempt %d/%d): %s. Retrying in %ds...", attempt+1, max_retries, e, delay)
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(delay)
|
||||
_monitor_logger.error("Docker monitor failed after %d attempts", max_retries)
|
||||
return None
|
||||
|
||||
client = docker.DockerClient(base_url=DOCKER_HOST)
|
||||
|
||||
# Store initial states
|
||||
@@ -186,7 +202,9 @@ async def monitor_containers():
|
||||
|
||||
while True:
|
||||
try:
|
||||
containers = await asyncio.to_thread(client.containers.list, all=True)
|
||||
containers = await asyncio.to_thread(
|
||||
lambda: docker_retry(lambda: client.containers.list(all=True)) or []
|
||||
)
|
||||
for c in containers:
|
||||
current_status = c.status
|
||||
if c.id in previous_states:
|
||||
|
||||
Reference in New Issue
Block a user