fix: improve terminal robustness with exponential backoff and better keepalive #34
@@ -41,8 +41,8 @@ TELEGRAM_BOT_TOKEN = os.environ.get("TELEGRAM_BOT_TOKEN", "")
|
|||||||
TELEGRAM_CHAT_ID = os.environ.get("TELEGRAM_CHAT_ID", "")
|
TELEGRAM_CHAT_ID = os.environ.get("TELEGRAM_CHAT_ID", "")
|
||||||
TELEGRAM_PROXY = os.environ.get("TELEGRAM_PROXY", "")
|
TELEGRAM_PROXY = os.environ.get("TELEGRAM_PROXY", "")
|
||||||
|
|
||||||
WEBAUTHN_RP_ID = os.environ.get("WEBAUTHN_RP_ID", "nas.jimmygan.com")
|
WEBAUTHN_RP_ID = os.environ.get("WEBAUTHN_RP_ID", "jimmygan.com")
|
||||||
WEBAUTHN_ORIGINS = os.environ.get("WEBAUTHN_ORIGIN", "https://nas.jimmygan.com,https://nas.jimmygan.com:8443").split(",")
|
WEBAUTHN_ORIGINS = os.environ.get("WEBAUTHN_ORIGINS", "https://nas.jimmygan.com,https://nas.jimmygan.com:8443").split(",")
|
||||||
|
|
||||||
# CORS
|
# CORS
|
||||||
CORS_ORIGINS = os.environ.get("CORS_ORIGINS", "https://nas.jimmygan.com,https://nas.jimmygan.com:8443").split(",")
|
CORS_ORIGINS = os.environ.get("CORS_ORIGINS", "https://nas.jimmygan.com,https://nas.jimmygan.com:8443").split(",")
|
||||||
|
|||||||
@@ -125,13 +125,22 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
|
|||||||
|
|
||||||
h = HOSTS[host]
|
h = HOSTS[host]
|
||||||
try:
|
try:
|
||||||
conn = await asyncssh.connect(
|
conn = await asyncio.wait_for(
|
||||||
h["host"], username=h["user"],
|
asyncssh.connect(
|
||||||
client_keys=[h["key"]], known_hosts=_known_hosts,
|
h["host"], username=h["user"],
|
||||||
keepalive_interval=30,
|
client_keys=[h["key"]], known_hosts=_known_hosts,
|
||||||
keepalive_count_max=3,
|
keepalive_interval=15,
|
||||||
|
keepalive_count_max=8,
|
||||||
|
),
|
||||||
|
timeout=10.0
|
||||||
)
|
)
|
||||||
except Exception:
|
except asyncio.TimeoutError:
|
||||||
|
logger.error("SSH connection to %s timed out after 10s", host)
|
||||||
|
await _release_session(session_id)
|
||||||
|
await websocket.close(code=1011, reason="SSH connection timeout")
|
||||||
|
return
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("SSH connection to %s failed: %s", host, e)
|
||||||
await _release_session(session_id)
|
await _release_session(session_id)
|
||||||
await websocket.close(code=1011, reason="SSH connection failed")
|
await websocket.close(code=1011, reason="SSH connection failed")
|
||||||
return
|
return
|
||||||
@@ -149,8 +158,12 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
|
|||||||
if not data:
|
if not data:
|
||||||
break
|
break
|
||||||
await websocket.send_bytes(data)
|
await websocket.send_bytes(data)
|
||||||
except (asyncssh.Error, asyncio.CancelledError):
|
except asyncssh.Error as e:
|
||||||
|
logger.warning("SSH read error for %s: %s", host, e)
|
||||||
|
except asyncio.CancelledError:
|
||||||
pass
|
pass
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("Unexpected error reading SSH for %s: %s", host, e)
|
||||||
|
|
||||||
read_task = asyncio.create_task(read_ssh())
|
read_task = asyncio.create_task(read_ssh())
|
||||||
|
|
||||||
@@ -169,8 +182,8 @@ async def ws_endpoint(websocket: WebSocket, host: str = Query("nas"), token: str
|
|||||||
if msg["text"] == "__ping__":
|
if msg["text"] == "__ping__":
|
||||||
continue
|
continue
|
||||||
process.stdin.write(msg["text"].encode())
|
process.stdin.write(msg["text"].encode())
|
||||||
except Exception:
|
except Exception as e:
|
||||||
pass
|
logger.info("WebSocket receive loop ended for %s: %s", host, e)
|
||||||
finally:
|
finally:
|
||||||
read_task.cancel()
|
read_task.cancel()
|
||||||
process.close()
|
process.close()
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ services:
|
|||||||
- TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID:-}
|
- TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID:-}
|
||||||
- TELEGRAM_PROXY=${TELEGRAM_PROXY:-}
|
- TELEGRAM_PROXY=${TELEGRAM_PROXY:-}
|
||||||
- CORS_ORIGINS=https://dev.nas.jimmygan.com,https://dev.nas.jimmygan.com:8443
|
- CORS_ORIGINS=https://dev.nas.jimmygan.com,https://dev.nas.jimmygan.com:8443
|
||||||
|
- WEBAUTHN_RP_ID=jimmygan.com
|
||||||
|
- WEBAUTHN_ORIGINS=https://dev.nas.jimmygan.com,https://dev.nas.jimmygan.com:8443,https://auth.jimmygan.com:8443
|
||||||
- LITELLM_URL=http://litellm:4005
|
- LITELLM_URL=http://litellm:4005
|
||||||
- LITELLM_HEALTH_API_KEY=${LITELLM_HEALTH_API_KEY:-}
|
- LITELLM_HEALTH_API_KEY=${LITELLM_HEALTH_API_KEY:-}
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
@@ -47,6 +47,8 @@ services:
|
|||||||
- TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID:-}
|
- TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID:-}
|
||||||
- TELEGRAM_PROXY=${TELEGRAM_PROXY:-}
|
- TELEGRAM_PROXY=${TELEGRAM_PROXY:-}
|
||||||
- CORS_ORIGINS=https://nas.jimmygan.com
|
- CORS_ORIGINS=https://nas.jimmygan.com
|
||||||
|
- WEBAUTHN_RP_ID=jimmygan.com
|
||||||
|
- WEBAUTHN_ORIGINS=https://nas.jimmygan.com,https://auth.jimmygan.com:8443
|
||||||
- LITELLM_HEALTH_API_KEY=${LITELLM_HEALTH_API_KEY:-}
|
- LITELLM_HEALTH_API_KEY=${LITELLM_HEALTH_API_KEY:-}
|
||||||
- LITELLM_URL=http://litellm:4005
|
- LITELLM_URL=http://litellm:4005
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
@@ -23,19 +23,46 @@ export function setRefreshToken() {}
|
|||||||
|
|
||||||
let refreshPromise = null;
|
let refreshPromise = null;
|
||||||
|
|
||||||
|
function getLegacyRefreshToken() {
|
||||||
|
return localStorage.getItem("refresh_token") || "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function clearLegacyRefreshToken() {
|
||||||
|
localStorage.removeItem("refresh_token");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestRefresh(body) {
|
||||||
|
const r = await fetch(BASE + "/auth/refresh", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
credentials: "same-origin",
|
||||||
|
body: body ? JSON.stringify(body) : undefined,
|
||||||
|
});
|
||||||
|
if (!r.ok) return false;
|
||||||
|
const data = await r.json();
|
||||||
|
setToken(data.access_token || "");
|
||||||
|
return !!data.access_token;
|
||||||
|
}
|
||||||
|
|
||||||
export async function tryRefreshSession() {
|
export async function tryRefreshSession() {
|
||||||
if (refreshPromise) return refreshPromise;
|
if (refreshPromise) return refreshPromise;
|
||||||
refreshPromise = (async () => {
|
refreshPromise = (async () => {
|
||||||
try {
|
try {
|
||||||
const r = await fetch(BASE + "/auth/refresh", {
|
const cookieRefreshed = await requestRefresh();
|
||||||
method: "POST",
|
if (cookieRefreshed) {
|
||||||
headers: { "Content-Type": "application/json" },
|
clearLegacyRefreshToken();
|
||||||
credentials: "same-origin",
|
return true;
|
||||||
});
|
}
|
||||||
if (!r.ok) return false;
|
|
||||||
const data = await r.json();
|
const legacyRefreshToken = getLegacyRefreshToken();
|
||||||
setToken(data.access_token || "");
|
if (!legacyRefreshToken) return false;
|
||||||
return !!data.access_token;
|
const legacyRefreshed = await requestRefresh({ refresh_token: legacyRefreshToken });
|
||||||
|
if (legacyRefreshed) {
|
||||||
|
clearLegacyRefreshToken();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
clearLegacyRefreshToken();
|
||||||
|
return false;
|
||||||
} catch {
|
} catch {
|
||||||
return false;
|
return false;
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -36,19 +36,27 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function registerPasskey() {
|
async function registerPasskey() {
|
||||||
|
alert("registerPasskey called!");
|
||||||
passkeyLoading = true;
|
passkeyLoading = true;
|
||||||
passkeyMsg = ""; passkeyErr = "";
|
passkeyMsg = ""; passkeyErr = "";
|
||||||
try {
|
try {
|
||||||
|
alert("Checking PublicKeyCredential support...");
|
||||||
if (!window.PublicKeyCredential) {
|
if (!window.PublicKeyCredential) {
|
||||||
|
alert("PublicKeyCredential NOT supported!");
|
||||||
throw new Error("Passkeys are not supported in this browser or context");
|
throw new Error("Passkeys are not supported in this browser or context");
|
||||||
}
|
}
|
||||||
|
alert("Fetching registration options...");
|
||||||
const opts = await post("/auth/passkey/register/options");
|
const opts = await post("/auth/passkey/register/options");
|
||||||
|
alert("Got options: " + JSON.stringify(opts).substring(0, 100));
|
||||||
opts.challenge = base64urlToBuffer(opts.challenge);
|
opts.challenge = base64urlToBuffer(opts.challenge);
|
||||||
opts.user.id = base64urlToBuffer(opts.user.id);
|
opts.user.id = base64urlToBuffer(opts.user.id);
|
||||||
if (opts.excludeCredentials) {
|
if (opts.excludeCredentials) {
|
||||||
opts.excludeCredentials = opts.excludeCredentials.map(c => ({ ...c, id: base64urlToBuffer(c.id) }));
|
opts.excludeCredentials = opts.excludeCredentials.map(c => ({ ...c, id: base64urlToBuffer(c.id) }));
|
||||||
}
|
}
|
||||||
const cred = await navigator.credentials.create({ publicKey: opts });
|
const cred = await navigator.credentials.create({ publicKey: opts });
|
||||||
|
if (!cred) {
|
||||||
|
throw new Error("Passkey creation was cancelled");
|
||||||
|
}
|
||||||
const name = prompt("Name this passkey (e.g. MacBook, iPhone):", "Passkey") || "Passkey";
|
const name = prompt("Name this passkey (e.g. MacBook, iPhone):", "Passkey") || "Passkey";
|
||||||
const body = {
|
const body = {
|
||||||
id: cred.id,
|
id: cred.id,
|
||||||
@@ -64,9 +72,13 @@
|
|||||||
passkeyMsg = "Passkey registered successfully";
|
passkeyMsg = "Passkey registered successfully";
|
||||||
await loadPasskeys();
|
await loadPasskeys();
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
passkeyErr = e.body?.detail || e.message || "Failed to register passkey";
|
console.error("Passkey registration error:", e);
|
||||||
|
const errorMsg = e.body?.detail || e.message || "Failed to register passkey";
|
||||||
|
passkeyErr = errorMsg;
|
||||||
|
alert("Passkey Error: " + errorMsg + "\n\nFull error: " + JSON.stringify(e, null, 2));
|
||||||
|
} finally {
|
||||||
|
passkeyLoading = false;
|
||||||
}
|
}
|
||||||
passkeyLoading = false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function clearPasskeys() {
|
async function clearPasskeys() {
|
||||||
|
|||||||
@@ -194,6 +194,8 @@
|
|||||||
let closedManually = false;
|
let closedManually = false;
|
||||||
let reconnecting = false;
|
let reconnecting = false;
|
||||||
let authRecoveryInFlight = false;
|
let authRecoveryInFlight = false;
|
||||||
|
let reconnectAttempts = 0;
|
||||||
|
const MAX_RECONNECT_DELAY = 30000;
|
||||||
|
|
||||||
function clearHeartbeat() {
|
function clearHeartbeat() {
|
||||||
if (heartbeat) {
|
if (heartbeat) {
|
||||||
@@ -229,17 +231,19 @@
|
|||||||
clearHeartbeat();
|
clearHeartbeat();
|
||||||
if (closedManually || reconnectTimer) return;
|
if (closedManually || reconnectTimer) return;
|
||||||
reconnecting = true;
|
reconnecting = true;
|
||||||
|
reconnectAttempts++;
|
||||||
|
const delay = Math.min(1000 * Math.pow(2, reconnectAttempts - 1), MAX_RECONNECT_DELAY);
|
||||||
updateTab(tabId, {
|
updateTab(tabId, {
|
||||||
connected: false,
|
connected: false,
|
||||||
error: `${reason} — reconnecting...`,
|
error: `${reason} — reconnecting in ${Math.round(delay / 1000)}s (attempt ${reconnectAttempts})...`,
|
||||||
statusBanner: tab.persistent ? "Reconnecting to persistent session..." : "",
|
statusBanner: tab.persistent ? "Reconnecting to persistent session..." : "",
|
||||||
});
|
});
|
||||||
term.write(`\r\n\x1b[90m[${reason} — reconnecting...]\x1b[0m`);
|
term.write(`\r\n\x1b[90m[${reason} — reconnecting in ${Math.round(delay / 1000)}s...]\x1b[0m`);
|
||||||
reconnectTimer = setTimeout(() => {
|
reconnectTimer = setTimeout(() => {
|
||||||
reconnectTimer = null;
|
reconnectTimer = null;
|
||||||
if (!tabData.has(tabId) || closedManually) return;
|
if (!tabData.has(tabId) || closedManually) return;
|
||||||
connect(true);
|
connect(true);
|
||||||
}, 2000);
|
}, delay);
|
||||||
}
|
}
|
||||||
|
|
||||||
function sendSize(ws) {
|
function sendSize(ws) {
|
||||||
@@ -267,6 +271,7 @@
|
|||||||
ws.binaryType = "arraybuffer";
|
ws.binaryType = "arraybuffer";
|
||||||
|
|
||||||
ws.onopen = () => {
|
ws.onopen = () => {
|
||||||
|
reconnectAttempts = 0;
|
||||||
updateTab(tabId, {
|
updateTab(tabId, {
|
||||||
connected: true,
|
connected: true,
|
||||||
error: "",
|
error: "",
|
||||||
@@ -276,7 +281,7 @@
|
|||||||
clearHeartbeat();
|
clearHeartbeat();
|
||||||
heartbeat = setInterval(() => {
|
heartbeat = setInterval(() => {
|
||||||
if (ws.readyState === 1) ws.send("__ping__");
|
if (ws.readyState === 1) ws.send("__ping__");
|
||||||
}, 25000);
|
}, 12000);
|
||||||
const d = tabData.get(tabId);
|
const d = tabData.get(tabId);
|
||||||
if (d) {
|
if (d) {
|
||||||
d.ws = ws;
|
d.ws = ws;
|
||||||
@@ -311,7 +316,7 @@
|
|||||||
authRecoveryInFlight = true;
|
authRecoveryInFlight = true;
|
||||||
const refreshed = await tryRefreshSession();
|
const refreshed = await tryRefreshSession();
|
||||||
authRecoveryInFlight = false;
|
authRecoveryInFlight = false;
|
||||||
if (refreshed) {
|
if (refreshed && !closedManually) {
|
||||||
reconnecting = true;
|
reconnecting = true;
|
||||||
updateTab(tabId, {
|
updateTab(tabId, {
|
||||||
connected: false,
|
connected: false,
|
||||||
|
|||||||
Reference in New Issue
Block a user