fix: improve terminal robustness with exponential backoff and better keepalive #34
@@ -125,13 +125,22 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
|
||||
|
||||
h = HOSTS[host]
|
||||
try:
|
||||
conn = await asyncssh.connect(
|
||||
conn = await asyncio.wait_for(
|
||||
asyncssh.connect(
|
||||
h["host"], username=h["user"],
|
||||
client_keys=[h["key"]], known_hosts=_known_hosts,
|
||||
keepalive_interval=30,
|
||||
keepalive_count_max=3,
|
||||
keepalive_interval=15,
|
||||
keepalive_count_max=8,
|
||||
),
|
||||
timeout=10.0
|
||||
)
|
||||
except Exception:
|
||||
except asyncio.TimeoutError:
|
||||
logger.error("SSH connection to %s timed out after 10s", host)
|
||||
await _release_session(session_id)
|
||||
await websocket.close(code=1011, reason="SSH connection timeout")
|
||||
return
|
||||
except Exception as e:
|
||||
logger.error("SSH connection to %s failed: %s", host, e)
|
||||
await _release_session(session_id)
|
||||
await websocket.close(code=1011, reason="SSH connection failed")
|
||||
return
|
||||
@@ -149,8 +158,12 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
|
||||
if not data:
|
||||
break
|
||||
await websocket.send_bytes(data)
|
||||
except (asyncssh.Error, asyncio.CancelledError):
|
||||
except asyncssh.Error as e:
|
||||
logger.warning("SSH read error for %s: %s", host, e)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error("Unexpected error reading SSH for %s: %s", host, e)
|
||||
|
||||
read_task = asyncio.create_task(read_ssh())
|
||||
|
||||
@@ -169,8 +182,8 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
|
||||
if msg["text"] == "__ping__":
|
||||
continue
|
||||
process.stdin.write(msg["text"].encode())
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.info("WebSocket receive loop ended for %s: %s", host, e)
|
||||
finally:
|
||||
read_task.cancel()
|
||||
process.close()
|
||||
|
||||
@@ -194,6 +194,8 @@
|
||||
let closedManually = false;
|
||||
let reconnecting = false;
|
||||
let authRecoveryInFlight = false;
|
||||
let reconnectAttempts = 0;
|
||||
const MAX_RECONNECT_DELAY = 30000;
|
||||
|
||||
function clearHeartbeat() {
|
||||
if (heartbeat) {
|
||||
@@ -229,17 +231,19 @@
|
||||
clearHeartbeat();
|
||||
if (closedManually || reconnectTimer) return;
|
||||
reconnecting = true;
|
||||
reconnectAttempts++;
|
||||
const delay = Math.min(1000 * Math.pow(2, reconnectAttempts - 1), MAX_RECONNECT_DELAY);
|
||||
updateTab(tabId, {
|
||||
connected: false,
|
||||
error: `${reason} — reconnecting...`,
|
||||
error: `${reason} — reconnecting in ${Math.round(delay / 1000)}s (attempt ${reconnectAttempts})...`,
|
||||
statusBanner: tab.persistent ? "Reconnecting to persistent session..." : "",
|
||||
});
|
||||
term.write(`\r\n\x1b[90m[${reason} — reconnecting...]\x1b[0m`);
|
||||
term.write(`\r\n\x1b[90m[${reason} — reconnecting in ${Math.round(delay / 1000)}s...]\x1b[0m`);
|
||||
reconnectTimer = setTimeout(() => {
|
||||
reconnectTimer = null;
|
||||
if (!tabData.has(tabId) || closedManually) return;
|
||||
connect(true);
|
||||
}, 2000);
|
||||
}, delay);
|
||||
}
|
||||
|
||||
function sendSize(ws) {
|
||||
@@ -267,6 +271,7 @@
|
||||
ws.binaryType = "arraybuffer";
|
||||
|
||||
ws.onopen = () => {
|
||||
reconnectAttempts = 0;
|
||||
updateTab(tabId, {
|
||||
connected: true,
|
||||
error: "",
|
||||
@@ -276,7 +281,7 @@
|
||||
clearHeartbeat();
|
||||
heartbeat = setInterval(() => {
|
||||
if (ws.readyState === 1) ws.send("__ping__");
|
||||
}, 25000);
|
||||
}, 12000);
|
||||
const d = tabData.get(tabId);
|
||||
if (d) {
|
||||
d.ws = ws;
|
||||
@@ -311,7 +316,7 @@
|
||||
authRecoveryInFlight = true;
|
||||
const refreshed = await tryRefreshSession();
|
||||
authRecoveryInFlight = false;
|
||||
if (refreshed) {
|
||||
if (refreshed && !closedManually) {
|
||||
reconnecting = true;
|
||||
updateTab(tabId, {
|
||||
connected: false,
|
||||
|
||||
Reference in New Issue
Block a user