From cc3e9dc6fa9c96c111fadd49553c822b5c81f7c0 Mon Sep 17 00:00:00 2001
From: "Gan, Jimmy" <jimmy.gan@adidas.com>
Date: Wed, 11 Mar 2026 00:24:17 +0800
Subject: [PATCH] fix: improve terminal robustness with exponential backoff and
 better keepalive
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Backend: Add 10s connection timeout, improve SSH keepalive (15s × 8 retries)
- Backend: Add detailed error logging for debugging
- Frontend: Implement exponential backoff (1s → 30s max) instead of fixed 2s
- Frontend: Reduce heartbeat to 12s (within 15s SSH keepalive window)
- Frontend: Reset reconnect counter on successful connection
- Frontend: Fix race condition in auth recovery after manual close
---
 dashboard/backend/routers/terminal.py         | 31 +++++++++++++------
 dashboard/frontend/src/routes/Terminal.svelte | 15 ++++++---
 2 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/dashboard/backend/routers/terminal.py b/dashboard/backend/routers/terminal.py
index e6134bd..8750a02 100644
--- a/dashboard/backend/routers/terminal.py
+++ b/dashboard/backend/routers/terminal.py
@@ -125,13 +125,22 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
 
     h = HOSTS[host]
     try:
-        conn = await asyncssh.connect(
-            h["host"], username=h["user"],
-            client_keys=[h["key"]], known_hosts=_known_hosts,
-            keepalive_interval=30,
-            keepalive_count_max=3,
+        conn = await asyncio.wait_for(
+            asyncssh.connect(
+                h["host"], username=h["user"],
+                client_keys=[h["key"]], known_hosts=_known_hosts,
+                keepalive_interval=15,
+                keepalive_count_max=8,
+            ),
+            timeout=10.0
         )
-    except Exception:
+    except asyncio.TimeoutError:
+        logger.error("SSH connection to %s timed out after 10s", host)
+        await _release_session(session_id)
+        await websocket.close(code=1011, reason="SSH connection timeout")
+        return
+    except Exception as e:
+        logger.error("SSH connection to %s failed: %s", host, e)
         await _release_session(session_id)
         await websocket.close(code=1011, reason="SSH connection failed")
         return
@@ -149,8 +158,12 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
                     if not data:
                         break
                     await websocket.send_bytes(data)
-            except (asyncssh.Error, asyncio.CancelledError):
+            except asyncssh.Error as e:
+                logger.warning("SSH read error for %s: %s", host, e)
+            except asyncio.CancelledError:
                 pass
+            except Exception as e:
+                logger.error("Unexpected error reading SSH for %s: %s", host, e)
 
         read_task = asyncio.create_task(read_ssh())
 
@@ -169,8 +182,8 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
                     if msg["text"] == "__ping__":
                         continue
                     process.stdin.write(msg["text"].encode())
-        except Exception:
-            pass
+        except Exception as e:
+            logger.info("WebSocket receive loop ended for %s: %s", host, e)
         finally:
             read_task.cancel()
             process.close()
diff --git a/dashboard/frontend/src/routes/Terminal.svelte b/dashboard/frontend/src/routes/Terminal.svelte
index 60f5bc4..273147c 100644
--- a/dashboard/frontend/src/routes/Terminal.svelte
+++ b/dashboard/frontend/src/routes/Terminal.svelte
@@ -194,6 +194,8 @@
       let closedManually = false;
       let reconnecting = false;
       let authRecoveryInFlight = false;
+      let reconnectAttempts = 0;
+      const MAX_RECONNECT_DELAY = 30000;
 
       function clearHeartbeat() {
         if (heartbeat) {
@@ -229,17 +231,19 @@
         clearHeartbeat();
         if (closedManually || reconnectTimer) return;
         reconnecting = true;
+        reconnectAttempts++;
+        const delay = Math.min(1000 * Math.pow(2, reconnectAttempts - 1), MAX_RECONNECT_DELAY);
         updateTab(tabId, {
           connected: false,
-          error: `${reason} — reconnecting...`,
+          error: `${reason} — reconnecting in ${Math.round(delay / 1000)}s (attempt ${reconnectAttempts})...`,
           statusBanner: tab.persistent ? "Reconnecting to persistent session..." : "",
         });
-        term.write(`\r\n\x1b[90m[${reason} — reconnecting...]\x1b[0m`);
+        term.write(`\r\n\x1b[90m[${reason} — reconnecting in ${Math.round(delay / 1000)}s...]\x1b[0m`);
         reconnectTimer = setTimeout(() => {
           reconnectTimer = null;
           if (!tabData.has(tabId) || closedManually) return;
           connect(true);
-        }, 2000);
+        }, delay);
       }
 
       function sendSize(ws) {
@@ -267,6 +271,7 @@
         ws.binaryType = "arraybuffer";
 
         ws.onopen = () => {
+          reconnectAttempts = 0;
           updateTab(tabId, {
             connected: true,
             error: "",
@@ -276,7 +281,7 @@
           clearHeartbeat();
           heartbeat = setInterval(() => {
             if (ws.readyState === 1) ws.send("__ping__");
-          }, 25000);
+          }, 12000);
           const d = tabData.get(tabId);
           if (d) {
             d.ws = ws;
@@ -311,7 +316,7 @@
               authRecoveryInFlight = true;
               const refreshed = await tryRefreshSession();
               authRecoveryInFlight = false;
-              if (refreshed) {
+              if (refreshed && !closedManually) {
                 reconnecting = true;
                 updateTab(tabId, {
                   connected: false,