fix: improve terminal robustness with exponential backoff and better keepalive #34

Merged
jimmy merged 9 commits from dev into main 2026-03-11 23:34:37 +08:00
2 changed files with 32 additions and 14 deletions
Showing only changes of commit cc3e9dc6fa - Show all commits
+20 -7
View File
@@ -125,13 +125,22 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
h = HOSTS[host] h = HOSTS[host]
try: try:
conn = await asyncssh.connect( conn = await asyncio.wait_for(
asyncssh.connect(
h["host"], username=h["user"], h["host"], username=h["user"],
client_keys=[h["key"]], known_hosts=_known_hosts, client_keys=[h["key"]], known_hosts=_known_hosts,
keepalive_interval=30, keepalive_interval=15,
keepalive_count_max=3, keepalive_count_max=8,
),
timeout=10.0
) )
except Exception: except asyncio.TimeoutError:
logger.error("SSH connection to %s timed out after 10s", host)
await _release_session(session_id)
await websocket.close(code=1011, reason="SSH connection timeout")
return
except Exception as e:
logger.error("SSH connection to %s failed: %s", host, e)
await _release_session(session_id) await _release_session(session_id)
await websocket.close(code=1011, reason="SSH connection failed") await websocket.close(code=1011, reason="SSH connection failed")
return return
@@ -149,8 +158,12 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
if not data: if not data:
break break
await websocket.send_bytes(data) await websocket.send_bytes(data)
except (asyncssh.Error, asyncio.CancelledError): except asyncssh.Error as e:
logger.warning("SSH read error for %s: %s", host, e)
except asyncio.CancelledError:
pass pass
except Exception as e:
logger.error("Unexpected error reading SSH for %s: %s", host, e)
read_task = asyncio.create_task(read_ssh()) read_task = asyncio.create_task(read_ssh())
@@ -169,8 +182,8 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
if msg["text"] == "__ping__": if msg["text"] == "__ping__":
continue continue
process.stdin.write(msg["text"].encode()) process.stdin.write(msg["text"].encode())
except Exception: except Exception as e:
pass logger.info("WebSocket receive loop ended for %s: %s", host, e)
finally: finally:
read_task.cancel() read_task.cancel()
process.close() process.close()
+10 -5
View File
@@ -194,6 +194,8 @@
let closedManually = false; let closedManually = false;
let reconnecting = false; let reconnecting = false;
let authRecoveryInFlight = false; let authRecoveryInFlight = false;
let reconnectAttempts = 0;
const MAX_RECONNECT_DELAY = 30000;
function clearHeartbeat() { function clearHeartbeat() {
if (heartbeat) { if (heartbeat) {
@@ -229,17 +231,19 @@
clearHeartbeat(); clearHeartbeat();
if (closedManually || reconnectTimer) return; if (closedManually || reconnectTimer) return;
reconnecting = true; reconnecting = true;
reconnectAttempts++;
const delay = Math.min(1000 * Math.pow(2, reconnectAttempts - 1), MAX_RECONNECT_DELAY);
updateTab(tabId, { updateTab(tabId, {
connected: false, connected: false,
error: `${reason} — reconnecting...`, error: `${reason} — reconnecting in ${Math.round(delay / 1000)}s (attempt ${reconnectAttempts})...`,
statusBanner: tab.persistent ? "Reconnecting to persistent session..." : "", statusBanner: tab.persistent ? "Reconnecting to persistent session..." : "",
}); });
term.write(`\r\n\x1b[90m[${reason} — reconnecting...]\x1b[0m`); term.write(`\r\n\x1b[90m[${reason} — reconnecting in ${Math.round(delay / 1000)}s...]\x1b[0m`);
reconnectTimer = setTimeout(() => { reconnectTimer = setTimeout(() => {
reconnectTimer = null; reconnectTimer = null;
if (!tabData.has(tabId) || closedManually) return; if (!tabData.has(tabId) || closedManually) return;
connect(true); connect(true);
}, 2000); }, delay);
} }
function sendSize(ws) { function sendSize(ws) {
@@ -267,6 +271,7 @@
ws.binaryType = "arraybuffer"; ws.binaryType = "arraybuffer";
ws.onopen = () => { ws.onopen = () => {
reconnectAttempts = 0;
updateTab(tabId, { updateTab(tabId, {
connected: true, connected: true,
error: "", error: "",
@@ -276,7 +281,7 @@
clearHeartbeat(); clearHeartbeat();
heartbeat = setInterval(() => { heartbeat = setInterval(() => {
if (ws.readyState === 1) ws.send("__ping__"); if (ws.readyState === 1) ws.send("__ping__");
}, 25000); }, 12000);
const d = tabData.get(tabId); const d = tabData.get(tabId);
if (d) { if (d) {
d.ws = ws; d.ws = ws;
@@ -311,7 +316,7 @@
authRecoveryInFlight = true; authRecoveryInFlight = true;
const refreshed = await tryRefreshSession(); const refreshed = await tryRefreshSession();
authRecoveryInFlight = false; authRecoveryInFlight = false;
if (refreshed) { if (refreshed && !closedManually) {
reconnecting = true; reconnecting = true;
updateTab(tabId, { updateTab(tabId, {
connected: false, connected: false,