From 00b5d362e7546168aaef7b1a01b6838d8f694322 Mon Sep 17 00:00:00 2001 From: "Gan, Jimmy" Date: Thu, 12 Mar 2026 00:23:23 +0800 Subject: [PATCH 1/5] debug: add detailed logging for terminal WebSocket connections --- dashboard/backend/routers/terminal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dashboard/backend/routers/terminal.py b/dashboard/backend/routers/terminal.py index 90ea7b6..d6482ad 100644 --- a/dashboard/backend/routers/terminal.py +++ b/dashboard/backend/routers/terminal.py @@ -122,6 +122,7 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str session_id = session_or_code await websocket.accept() + logger.info("WebSocket accepted for %s (user: %s)", host, user.username) h = HOSTS[host] try: @@ -134,6 +135,7 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str ), timeout=10.0 ) + logger.info("SSH connection established to %s", host) except asyncio.TimeoutError: logger.error("SSH connection to %s timed out after 10s", host) await _release_session(session_id) @@ -180,11 +182,12 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str process.stdin.write(data) elif "text" in msg and msg["text"]: if msg["text"] == "__ping__": + logger.debug("Received ping from %s, sending pong", host) await websocket.send_text("__pong__") continue process.stdin.write(msg["text"].encode()) except Exception as e: - logger.info("WebSocket receive loop ended for %s: %s", host, e) + logger.warning("WebSocket receive loop ended for %s: %s", host, e) finally: read_task.cancel() process.close() -- 2.52.0 From 7b17a53cab6ae6eb2a0a18249e1e73bb14f24882 Mon Sep 17 00:00:00 2001 From: "Gan, Jimmy" Date: Fri, 13 Mar 2026 22:26:22 +0800 Subject: [PATCH 2/5] fix: resolve terminal WebSocket auth failures and improve reconnection - Remove insecure query token parameter from backend (cookie-only auth) - Add detailed JWT error logging (expired, invalid, missing tokens) - Force token refresh before first WebSocket connection - Stop reconnection after 3 consecutive auth failures - Increase ping interval from 12s to 30s (reduce traffic) - Increase pong timeout from 5s to 10s (handle network latency) - Show attempt counter in reconnection error messages Root cause: Frontend was reconnecting with 6-day-old expired token, causing 403 rejections. Now ensures fresh cookies before connecting. --- dashboard/backend/routers/terminal.py | 17 ++++++++-- dashboard/frontend/src/routes/Terminal.svelte | 33 ++++++++++++++++--- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/dashboard/backend/routers/terminal.py b/dashboard/backend/routers/terminal.py index d6482ad..e61e08f 100644 --- a/dashboard/backend/routers/terminal.py +++ b/dashboard/backend/routers/terminal.py @@ -2,6 +2,7 @@ import asyncio import struct import logging import shlex +import jwt from collections import Counter from fastapi import WebSocket, Query import asyncssh @@ -90,19 +91,29 @@ async def _release_session(session_id: int | None): _active_sessions.pop(session_id, None) -async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str | None = Query(None)): +async def ws_endpoint(websocket: WebSocket, host: str = Query("nas")): if host not in HOSTS: await websocket.close(code=1008, reason="Unknown host") return - access_token = websocket.cookies.get(auth.ACCESS_COOKIE_NAME) or token + access_token = websocket.cookies.get(auth.ACCESS_COOKIE_NAME) if not access_token: + logger.warning("WebSocket auth failed for %s: no access token cookie", host) await websocket.close(code=1008, reason="Unauthorized") return try: user = await auth.get_current_user_ws(access_token) - except Exception: + except jwt.ExpiredSignatureError as e: + logger.warning("WebSocket auth failed for %s: token expired - %s", host, e) + await websocket.close(code=1008, reason="Token expired") + return + except jwt.InvalidTokenError as e: + logger.warning("WebSocket auth failed for %s: invalid token - %s", host, e) + await websocket.close(code=1008, reason="Invalid token") + return + except Exception as e: + logger.error("WebSocket auth failed for %s: unexpected error - %s", host, e) await websocket.close(code=1008, reason="Unauthorized") return diff --git a/dashboard/frontend/src/routes/Terminal.svelte b/dashboard/frontend/src/routes/Terminal.svelte index d297558..de75c1f 100644 --- a/dashboard/frontend/src/routes/Terminal.svelte +++ b/dashboard/frontend/src/routes/Terminal.svelte @@ -196,8 +196,11 @@ let reconnecting = false; let authRecoveryInFlight = false; let reconnectAttempts = 0; + let consecutiveAuthFailures = 0; const MAX_RECONNECT_DELAY = 30000; - const PONG_TIMEOUT = 5000; // 5s to receive pong + const MAX_AUTH_FAILURES = 3; + const PING_INTERVAL = 30000; // 30s between pings + const PONG_TIMEOUT = 10000; // 10s to receive pong function clearHeartbeat() { if (heartbeat) { @@ -263,12 +266,18 @@ } async function connect(forceRefresh = false) { - if (forceRefresh) { + // Always refresh token before first connection to ensure fresh cookies + const d = tabData.get(tabId); + const isFirstConnect = !d?.hasConnectedOnce; + + if (forceRefresh || isFirstConnect) { const refreshed = await tryRefreshSession(); if (!refreshed) { handleAuthExpired(); return null; } + // Mark that we've connected once with fresh token + if (d) d.hasConnectedOnce = true; } const ws = new WebSocket( @@ -278,6 +287,7 @@ ws.onopen = () => { reconnectAttempts = 0; + consecutiveAuthFailures = 0; // Reset auth failure counter on successful connection updateTab(tabId, { connected: true, error: "", @@ -295,7 +305,7 @@ } }, PONG_TIMEOUT); } - }, 12000); + }, PING_INTERVAL); const d = tabData.get(tabId); if (d) { d.ws = ws; @@ -335,7 +345,18 @@ const reason = e.reason || `Connection closed (code ${e.code})`; const d = tabData.get(tabId); if (d) d.ws = null; - if (e.code === 1008 && reason === "Unauthorized") { + + // Handle auth failures (code 1008) + if (e.code === 1008) { + consecutiveAuthFailures++; + + // Stop after MAX_AUTH_FAILURES consecutive auth failures + if (consecutiveAuthFailures >= MAX_AUTH_FAILURES) { + handleAuthExpired(`Session expired after ${MAX_AUTH_FAILURES} attempts — please refresh page`); + return; + } + + // Try to refresh session if (!closedManually && !authRecoveryInFlight) { authRecoveryInFlight = true; const refreshed = await tryRefreshSession(); @@ -344,7 +365,7 @@ reconnecting = true; updateTab(tabId, { connected: false, - error: "Refreshing session...", + error: `Refreshing session (attempt ${consecutiveAuthFailures}/${MAX_AUTH_FAILURES})...`, statusBanner: tab.persistent ? "Reconnecting to persistent session..." : "", }); term.write(`\r\n\x1b[90m[Refreshing session...]\x1b[0m`); @@ -355,6 +376,8 @@ handleAuthExpired("Session expired — please log in again"); return; } + + // Non-auth failures: continue with normal reconnection if (!closedManually) scheduleReconnect(reason); else { clearHeartbeat(); -- 2.52.0 From d69b744ae7f84298060d07bd20aad01d53e2876a Mon Sep 17 00:00:00 2001 From: "Gan, Jimmy" Date: Fri, 13 Mar 2026 22:45:55 +0800 Subject: [PATCH 3/5] fix: resolve variable redeclaration in Terminal.svelte Change 'const d' to 'let d' to allow reassignment in nested scopes --- dashboard/frontend/src/routes/Terminal.svelte | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboard/frontend/src/routes/Terminal.svelte b/dashboard/frontend/src/routes/Terminal.svelte index de75c1f..db3c6b8 100644 --- a/dashboard/frontend/src/routes/Terminal.svelte +++ b/dashboard/frontend/src/routes/Terminal.svelte @@ -267,7 +267,7 @@ async function connect(forceRefresh = false) { // Always refresh token before first connection to ensure fresh cookies - const d = tabData.get(tabId); + let d = tabData.get(tabId); const isFirstConnect = !d?.hasConnectedOnce; if (forceRefresh || isFirstConnect) { @@ -343,7 +343,7 @@ }; ws.onclose = async (e) => { const reason = e.reason || `Connection closed (code ${e.code})`; - const d = tabData.get(tabId); + d = tabData.get(tabId); if (d) d.ws = null; // Handle auth failures (code 1008) @@ -386,13 +386,13 @@ } }; ws.onerror = () => { - const d = tabData.get(tabId); + d = tabData.get(tabId); if (d) d.ws = null; if (!closedManually) scheduleReconnect("Connection failed"); else updateTab(tabId, { connected: false, error: "Connection failed" }); }; - const d = tabData.get(tabId); + d = tabData.get(tabId); if (d) { d.ws = ws; d.heartbeat = heartbeat; -- 2.52.0 From d0078f7e9d10554280b8da6b9ede8ab38fce7a86 Mon Sep 17 00:00:00 2001 From: "Gan, Jimmy" Date: Sun, 15 Mar 2026 00:22:05 +0800 Subject: [PATCH 4/5] fix: update Immich healthcheck to use curl and limit ffmpeg threads --- immich/docker-compose.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/immich/docker-compose.yml b/immich/docker-compose.yml index 5cc7de0..9ec7275 100644 --- a/immich/docker-compose.yml +++ b/immich/docker-compose.yml @@ -18,9 +18,10 @@ services: DB_PASSWORD: ${DB_PASSWORD} DB_DATABASE_NAME: ${DB_DATABASE_NAME} REDIS_HOSTNAME: immich-redis - MACHINE_LEARNING_URL: http://100.65.185.12:3003 + IMMICH_MACHINE_LEARNING_ENABLED: "false" + IMMICH_MEDIA_FFMPEG_THREADS: 2 healthcheck: - test: ["CMD", "wget", "-q", "--spider", "http://localhost:2283/api/server/ping"] + test: ["CMD-SHELL", "curl -f http://localhost:2283/api/server/ping || exit 1"] interval: 30s timeout: 5s retries: 3 -- 2.52.0 From 99295676799ed89c2e2f9eb29d04ef54ca7eadf7 Mon Sep 17 00:00:00 2001 From: "Gan, Jimmy" Date: Sun, 15 Mar 2026 00:29:45 +0800 Subject: [PATCH 5/5] fix: bind Immich to 0.0.0.0 to allow Tailscale access --- immich/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/immich/docker-compose.yml b/immich/docker-compose.yml index 9ec7275..0674111 100644 --- a/immich/docker-compose.yml +++ b/immich/docker-compose.yml @@ -6,7 +6,7 @@ services: labels: - "com.centurylinklabs.watchtower.enable=true" ports: - - "127.0.0.1:2283:2283" + - "0.0.0.0:2283:2283" volumes: - ${UPLOAD_LOCATION}:/usr/src/app/upload - /volume1/photo:/mnt/photo:ro -- 2.52.0