fix: improve terminal robustness with exponential backoff and better keepalive #34

Merged
jimmy merged 9 commits from dev into main 2026-03-11 23:34:37 +08:00
2 changed files with 32 additions and 14 deletions
Showing only changes of commit cc3e9dc6fa - Show all commits
+22 -9
View File
@@ -125,13 +125,22 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
h = HOSTS[host]
try:
conn = await asyncssh.connect(
h["host"], username=h["user"],
client_keys=[h["key"]], known_hosts=_known_hosts,
keepalive_interval=30,
keepalive_count_max=3,
conn = await asyncio.wait_for(
asyncssh.connect(
h["host"], username=h["user"],
client_keys=[h["key"]], known_hosts=_known_hosts,
keepalive_interval=15,
keepalive_count_max=8,
),
timeout=10.0
)
except Exception:
except asyncio.TimeoutError:
logger.error("SSH connection to %s timed out after 10s", host)
await _release_session(session_id)
await websocket.close(code=1011, reason="SSH connection timeout")
return
except Exception as e:
logger.error("SSH connection to %s failed: %s", host, e)
await _release_session(session_id)
await websocket.close(code=1011, reason="SSH connection failed")
return
@@ -149,8 +158,12 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
if not data:
break
await websocket.send_bytes(data)
except (asyncssh.Error, asyncio.CancelledError):
except asyncssh.Error as e:
logger.warning("SSH read error for %s: %s", host, e)
except asyncio.CancelledError:
pass
except Exception as e:
logger.error("Unexpected error reading SSH for %s: %s", host, e)
read_task = asyncio.create_task(read_ssh())
@@ -169,8 +182,8 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
if msg["text"] == "__ping__":
continue
process.stdin.write(msg["text"].encode())
except Exception:
pass
except Exception as e:
logger.info("WebSocket receive loop ended for %s: %s", host, e)
finally:
read_task.cancel()
process.close()
+10 -5
View File
@@ -194,6 +194,8 @@
let closedManually = false;
let reconnecting = false;
let authRecoveryInFlight = false;
let reconnectAttempts = 0;
const MAX_RECONNECT_DELAY = 30000;
function clearHeartbeat() {
if (heartbeat) {
@@ -229,17 +231,19 @@
clearHeartbeat();
if (closedManually || reconnectTimer) return;
reconnecting = true;
reconnectAttempts++;
const delay = Math.min(1000 * Math.pow(2, reconnectAttempts - 1), MAX_RECONNECT_DELAY);
updateTab(tabId, {
connected: false,
error: `${reason} — reconnecting...`,
error: `${reason} — reconnecting in ${Math.round(delay / 1000)}s (attempt ${reconnectAttempts})...`,
statusBanner: tab.persistent ? "Reconnecting to persistent session..." : "",
});
term.write(`\r\n\x1b[90m[${reason} — reconnecting...]\x1b[0m`);
term.write(`\r\n\x1b[90m[${reason} — reconnecting in ${Math.round(delay / 1000)}s...]\x1b[0m`);
reconnectTimer = setTimeout(() => {
reconnectTimer = null;
if (!tabData.has(tabId) || closedManually) return;
connect(true);
}, 2000);
}, delay);
}
function sendSize(ws) {
@@ -267,6 +271,7 @@
ws.binaryType = "arraybuffer";
ws.onopen = () => {
reconnectAttempts = 0;
updateTab(tabId, {
connected: true,
error: "",
@@ -276,7 +281,7 @@
clearHeartbeat();
heartbeat = setInterval(() => {
if (ws.readyState === 1) ws.send("__ping__");
}, 25000);
}, 12000);
const d = tabData.get(tabId);
if (d) {
d.ws = ws;
@@ -311,7 +316,7 @@
authRecoveryInFlight = true;
const refreshed = await tryRefreshSession();
authRecoveryInFlight = false;
if (refreshed) {
if (refreshed && !closedManually) {
reconnecting = true;
updateTab(tabId, {
connected: false,