fix: improve terminal robustness with exponential backoff and better keepalive #34

Merged
jimmy merged 9 commits from dev into main 2026-03-11 23:34:37 +08:00
7 changed files with 88 additions and 27 deletions
+2 -2
View File
@@ -41,8 +41,8 @@ TELEGRAM_BOT_TOKEN = os.environ.get("TELEGRAM_BOT_TOKEN", "")
TELEGRAM_CHAT_ID = os.environ.get("TELEGRAM_CHAT_ID", "") TELEGRAM_CHAT_ID = os.environ.get("TELEGRAM_CHAT_ID", "")
TELEGRAM_PROXY = os.environ.get("TELEGRAM_PROXY", "") TELEGRAM_PROXY = os.environ.get("TELEGRAM_PROXY", "")
WEBAUTHN_RP_ID = os.environ.get("WEBAUTHN_RP_ID", "nas.jimmygan.com") WEBAUTHN_RP_ID = os.environ.get("WEBAUTHN_RP_ID", "jimmygan.com")
WEBAUTHN_ORIGINS = os.environ.get("WEBAUTHN_ORIGIN", "https://nas.jimmygan.com,https://nas.jimmygan.com:8443").split(",") WEBAUTHN_ORIGINS = os.environ.get("WEBAUTHN_ORIGINS", "https://nas.jimmygan.com,https://nas.jimmygan.com:8443").split(",")
# CORS # CORS
CORS_ORIGINS = os.environ.get("CORS_ORIGINS", "https://nas.jimmygan.com,https://nas.jimmygan.com:8443").split(",") CORS_ORIGINS = os.environ.get("CORS_ORIGINS", "https://nas.jimmygan.com,https://nas.jimmygan.com:8443").split(",")
+22 -9
View File
@@ -125,13 +125,22 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
h = HOSTS[host] h = HOSTS[host]
try: try:
conn = await asyncssh.connect( conn = await asyncio.wait_for(
h["host"], username=h["user"], asyncssh.connect(
client_keys=[h["key"]], known_hosts=_known_hosts, h["host"], username=h["user"],
keepalive_interval=30, client_keys=[h["key"]], known_hosts=_known_hosts,
keepalive_count_max=3, keepalive_interval=15,
keepalive_count_max=8,
),
timeout=10.0
) )
except Exception: except asyncio.TimeoutError:
logger.error("SSH connection to %s timed out after 10s", host)
await _release_session(session_id)
await websocket.close(code=1011, reason="SSH connection timeout")
return
except Exception as e:
logger.error("SSH connection to %s failed: %s", host, e)
await _release_session(session_id) await _release_session(session_id)
await websocket.close(code=1011, reason="SSH connection failed") await websocket.close(code=1011, reason="SSH connection failed")
return return
@@ -149,8 +158,12 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
if not data: if not data:
break break
await websocket.send_bytes(data) await websocket.send_bytes(data)
except (asyncssh.Error, asyncio.CancelledError): except asyncssh.Error as e:
logger.warning("SSH read error for %s: %s", host, e)
except asyncio.CancelledError:
pass pass
except Exception as e:
logger.error("Unexpected error reading SSH for %s: %s", host, e)
read_task = asyncio.create_task(read_ssh()) read_task = asyncio.create_task(read_ssh())
@@ -169,8 +182,8 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
if msg["text"] == "__ping__": if msg["text"] == "__ping__":
continue continue
process.stdin.write(msg["text"].encode()) process.stdin.write(msg["text"].encode())
except Exception: except Exception as e:
pass logger.info("WebSocket receive loop ended for %s: %s", host, e)
finally: finally:
read_task.cancel() read_task.cancel()
process.close() process.close()
+2
View File
@@ -25,6 +25,8 @@ services:
- TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID:-} - TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID:-}
- TELEGRAM_PROXY=${TELEGRAM_PROXY:-} - TELEGRAM_PROXY=${TELEGRAM_PROXY:-}
- CORS_ORIGINS=https://dev.nas.jimmygan.com,https://dev.nas.jimmygan.com:8443 - CORS_ORIGINS=https://dev.nas.jimmygan.com,https://dev.nas.jimmygan.com:8443
- WEBAUTHN_RP_ID=jimmygan.com
- WEBAUTHN_ORIGINS=https://dev.nas.jimmygan.com,https://dev.nas.jimmygan.com:8443,https://auth.jimmygan.com:8443
- LITELLM_URL=http://litellm:4005 - LITELLM_URL=http://litellm:4005
- LITELLM_HEALTH_API_KEY=${LITELLM_HEALTH_API_KEY:-} - LITELLM_HEALTH_API_KEY=${LITELLM_HEALTH_API_KEY:-}
volumes: volumes:
+2
View File
@@ -47,6 +47,8 @@ services:
- TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID:-} - TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID:-}
- TELEGRAM_PROXY=${TELEGRAM_PROXY:-} - TELEGRAM_PROXY=${TELEGRAM_PROXY:-}
- CORS_ORIGINS=https://nas.jimmygan.com - CORS_ORIGINS=https://nas.jimmygan.com
- WEBAUTHN_RP_ID=jimmygan.com
- WEBAUTHN_ORIGINS=https://nas.jimmygan.com,https://auth.jimmygan.com:8443
- LITELLM_HEALTH_API_KEY=${LITELLM_HEALTH_API_KEY:-} - LITELLM_HEALTH_API_KEY=${LITELLM_HEALTH_API_KEY:-}
- LITELLM_URL=http://litellm:4005 - LITELLM_URL=http://litellm:4005
volumes: volumes:
+36 -9
View File
@@ -23,19 +23,46 @@ export function setRefreshToken() {}
let refreshPromise = null; let refreshPromise = null;
function getLegacyRefreshToken() {
return localStorage.getItem("refresh_token") || "";
}
function clearLegacyRefreshToken() {
localStorage.removeItem("refresh_token");
}
async function requestRefresh(body) {
const r = await fetch(BASE + "/auth/refresh", {
method: "POST",
headers: { "Content-Type": "application/json" },
credentials: "same-origin",
body: body ? JSON.stringify(body) : undefined,
});
if (!r.ok) return false;
const data = await r.json();
setToken(data.access_token || "");
return !!data.access_token;
}
export async function tryRefreshSession() { export async function tryRefreshSession() {
if (refreshPromise) return refreshPromise; if (refreshPromise) return refreshPromise;
refreshPromise = (async () => { refreshPromise = (async () => {
try { try {
const r = await fetch(BASE + "/auth/refresh", { const cookieRefreshed = await requestRefresh();
method: "POST", if (cookieRefreshed) {
headers: { "Content-Type": "application/json" }, clearLegacyRefreshToken();
credentials: "same-origin", return true;
}); }
if (!r.ok) return false;
const data = await r.json(); const legacyRefreshToken = getLegacyRefreshToken();
setToken(data.access_token || ""); if (!legacyRefreshToken) return false;
return !!data.access_token; const legacyRefreshed = await requestRefresh({ refresh_token: legacyRefreshToken });
if (legacyRefreshed) {
clearLegacyRefreshToken();
return true;
}
clearLegacyRefreshToken();
return false;
} catch { } catch {
return false; return false;
} finally { } finally {
+14 -2
View File
@@ -36,19 +36,27 @@
} }
async function registerPasskey() { async function registerPasskey() {
alert("registerPasskey called!");
passkeyLoading = true; passkeyLoading = true;
passkeyMsg = ""; passkeyErr = ""; passkeyMsg = ""; passkeyErr = "";
try { try {
alert("Checking PublicKeyCredential support...");
if (!window.PublicKeyCredential) { if (!window.PublicKeyCredential) {
alert("PublicKeyCredential NOT supported!");
throw new Error("Passkeys are not supported in this browser or context"); throw new Error("Passkeys are not supported in this browser or context");
} }
alert("Fetching registration options...");
const opts = await post("/auth/passkey/register/options"); const opts = await post("/auth/passkey/register/options");
alert("Got options: " + JSON.stringify(opts).substring(0, 100));
opts.challenge = base64urlToBuffer(opts.challenge); opts.challenge = base64urlToBuffer(opts.challenge);
opts.user.id = base64urlToBuffer(opts.user.id); opts.user.id = base64urlToBuffer(opts.user.id);
if (opts.excludeCredentials) { if (opts.excludeCredentials) {
opts.excludeCredentials = opts.excludeCredentials.map(c => ({ ...c, id: base64urlToBuffer(c.id) })); opts.excludeCredentials = opts.excludeCredentials.map(c => ({ ...c, id: base64urlToBuffer(c.id) }));
} }
const cred = await navigator.credentials.create({ publicKey: opts }); const cred = await navigator.credentials.create({ publicKey: opts });
if (!cred) {
throw new Error("Passkey creation was cancelled");
}
const name = prompt("Name this passkey (e.g. MacBook, iPhone):", "Passkey") || "Passkey"; const name = prompt("Name this passkey (e.g. MacBook, iPhone):", "Passkey") || "Passkey";
const body = { const body = {
id: cred.id, id: cred.id,
@@ -64,9 +72,13 @@
passkeyMsg = "Passkey registered successfully"; passkeyMsg = "Passkey registered successfully";
await loadPasskeys(); await loadPasskeys();
} catch (e) { } catch (e) {
passkeyErr = e.body?.detail || e.message || "Failed to register passkey"; console.error("Passkey registration error:", e);
const errorMsg = e.body?.detail || e.message || "Failed to register passkey";
passkeyErr = errorMsg;
alert("Passkey Error: " + errorMsg + "\n\nFull error: " + JSON.stringify(e, null, 2));
} finally {
passkeyLoading = false;
} }
passkeyLoading = false;
} }
async function clearPasskeys() { async function clearPasskeys() {
+10 -5
View File
@@ -194,6 +194,8 @@
let closedManually = false; let closedManually = false;
let reconnecting = false; let reconnecting = false;
let authRecoveryInFlight = false; let authRecoveryInFlight = false;
let reconnectAttempts = 0;
const MAX_RECONNECT_DELAY = 30000;
function clearHeartbeat() { function clearHeartbeat() {
if (heartbeat) { if (heartbeat) {
@@ -229,17 +231,19 @@
clearHeartbeat(); clearHeartbeat();
if (closedManually || reconnectTimer) return; if (closedManually || reconnectTimer) return;
reconnecting = true; reconnecting = true;
reconnectAttempts++;
const delay = Math.min(1000 * Math.pow(2, reconnectAttempts - 1), MAX_RECONNECT_DELAY);
updateTab(tabId, { updateTab(tabId, {
connected: false, connected: false,
error: `${reason} — reconnecting...`, error: `${reason} — reconnecting in ${Math.round(delay / 1000)}s (attempt ${reconnectAttempts})...`,
statusBanner: tab.persistent ? "Reconnecting to persistent session..." : "", statusBanner: tab.persistent ? "Reconnecting to persistent session..." : "",
}); });
term.write(`\r\n\x1b[90m[${reason} — reconnecting...]\x1b[0m`); term.write(`\r\n\x1b[90m[${reason} — reconnecting in ${Math.round(delay / 1000)}s...]\x1b[0m`);
reconnectTimer = setTimeout(() => { reconnectTimer = setTimeout(() => {
reconnectTimer = null; reconnectTimer = null;
if (!tabData.has(tabId) || closedManually) return; if (!tabData.has(tabId) || closedManually) return;
connect(true); connect(true);
}, 2000); }, delay);
} }
function sendSize(ws) { function sendSize(ws) {
@@ -267,6 +271,7 @@
ws.binaryType = "arraybuffer"; ws.binaryType = "arraybuffer";
ws.onopen = () => { ws.onopen = () => {
reconnectAttempts = 0;
updateTab(tabId, { updateTab(tabId, {
connected: true, connected: true,
error: "", error: "",
@@ -276,7 +281,7 @@
clearHeartbeat(); clearHeartbeat();
heartbeat = setInterval(() => { heartbeat = setInterval(() => {
if (ws.readyState === 1) ws.send("__ping__"); if (ws.readyState === 1) ws.send("__ping__");
}, 25000); }, 12000);
const d = tabData.get(tabId); const d = tabData.get(tabId);
if (d) { if (d) {
d.ws = ws; d.ws = ws;
@@ -311,7 +316,7 @@
authRecoveryInFlight = true; authRecoveryInFlight = true;
const refreshed = await tryRefreshSession(); const refreshed = await tryRefreshSession();
authRecoveryInFlight = false; authRecoveryInFlight = false;
if (refreshed) { if (refreshed && !closedManually) {
reconnecting = true; reconnecting = true;
updateTab(tabId, { updateTab(tabId, {
connected: false, connected: false,