From 88e53f3f8ee56b7752c7fa9025f9e31d0b30ce3f Mon Sep 17 00:00:00 2001
From: "Gan, Jimmy" <jimmy.gan@adidas.com>
Date: Sat, 18 Apr 2026 11:03:22 +0800
Subject: [PATCH] fix: implement streaming downloads for large files

- Add token-based download system with 5-minute expiry
- Stream files in 8MB chunks on backend (no memory loading)
- Use browser native download with tokens on frontend
- Fixes hang on large files (10GB+) by avoiding memory buffering
---
 dashboard/backend/routers/files.py | 68 +++++++++++++++++++++++++++---
 dashboard/frontend/src/lib/api.js  | 22 +++++-----
 2 files changed, 75 insertions(+), 15 deletions(-)

diff --git a/dashboard/backend/routers/files.py b/dashboard/backend/routers/files.py
index 3f58f87..20ea7b4 100644
--- a/dashboard/backend/routers/files.py
+++ b/dashboard/backend/routers/files.py
@@ -1,11 +1,13 @@
 import logging
 import os
 import re
+import secrets
 import shutil
+import time
 from pathlib import Path
 
-from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
-from fastapi.responses import FileResponse
+from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, Query
+from fastapi.responses import StreamingResponse
 
 from config import VOLUME_ROOT
 from rbac import require_admin
@@ -17,6 +19,10 @@ BASE = Path(VOLUME_ROOT)
 MAX_UPLOAD_SIZE = 100 * 1024 * 1024  # 100 MB
 BLOCKED_DIRS = {"@appstore", "@docker", "@eaDir", "@S2S", "@sharesnap", "@tmp", "docker", "homes", "backups"}
 
+# Download token storage: {token: (path, expiry_timestamp)}
+_download_tokens = {}
+TOKEN_EXPIRY = 300  # 5 minutes
+
 
 _BASE_RESOLVED = str(BASE.resolve())
 
@@ -88,13 +94,65 @@ def browse(path: str = ""):
     return {"path": str(target.relative_to(BASE)), "entries": entries}
 
 
-@router.get("/download")
-def download(path: str):
+@router.post("/download-token")
+def create_download_token(path: str):
+    """Generate a temporary download token for large file downloads."""
     try:
         target = _safe_path(path)
         if not target.exists():
             raise HTTPException(status_code=404, detail="File not found")
-        return FileResponse(target, filename=target.name)
+
+        # Clean up expired tokens
+        now = time.time()
+        expired = [t for t, (_, exp) in _download_tokens.items() if exp < now]
+        for t in expired:
+            del _download_tokens[t]
+
+        # Generate new token
+        token = secrets.token_urlsafe(32)
+        _download_tokens[token] = (str(target), now + TOKEN_EXPIRY)
+
+        return {"token": token, "expires_in": TOKEN_EXPIRY}
+    except ValueError:
+        raise HTTPException(status_code=403, detail="Access denied")
+
+
+@router.get("/download")
+def download(path: str = None, token: str = Query(None)):
+    """Download a file. Supports both authenticated access (path param) and token-based access (token param)."""
+    try:
+        if token:
+            # Token-based download (no auth required)
+            if token not in _download_tokens:
+                raise HTTPException(status_code=403, detail="Invalid or expired token")
+
+            file_path, expiry = _download_tokens[token]
+            if time.time() > expiry:
+                del _download_tokens[token]
+                raise HTTPException(status_code=403, detail="Token expired")
+
+            # Consume token after use
+            del _download_tokens[token]
+            target = Path(file_path)
+        else:
+            # Authenticated download
+            if not path:
+                raise HTTPException(status_code=400, detail="Path or token required")
+            target = _safe_path(path)
+
+        if not target.exists():
+            raise HTTPException(status_code=404, detail="File not found")
+
+        def file_iterator():
+            with open(target, "rb") as f:
+                while chunk := f.read(8192 * 1024):  # 8MB chunks
+                    yield chunk
+
+        return StreamingResponse(
+            file_iterator(),
+            media_type="application/octet-stream",
+            headers={"Content-Disposition": f'attachment; filename="{target.name}"'}
+        )
     except ValueError:
         raise HTTPException(status_code=403, detail="Access denied")
 
diff --git a/dashboard/frontend/src/lib/api.js b/dashboard/frontend/src/lib/api.js
index b54f049..864b0e0 100644
--- a/dashboard/frontend/src/lib/api.js
+++ b/dashboard/frontend/src/lib/api.js
@@ -203,25 +203,27 @@ export async function download(path, filename) {
   if (token) headers["Authorization"] = `Bearer ${token}`;
 
   try {
-    const r = await fetch(`${BASE}/files/download?path=${encodeURIComponent(path)}`, {
-      method: "GET",
+    // Get a temporary download token
+    const tokenResponse = await fetch(`${BASE}/files/download-token?path=${encodeURIComponent(path)}`, {
+      method: "POST",
       headers,
     });
 
-    if (!r.ok) {
-      const errorText = await r.text();
-      console.error(`Download failed: ${r.status} - ${errorText}`);
-      throw new Error(`Download failed: ${r.status}`);
+    if (!tokenResponse.ok) {
+      const errorText = await tokenResponse.text();
+      console.error(`Download token failed: ${tokenResponse.status} - ${errorText}`);
+      throw new Error(`Download failed: ${tokenResponse.status}`);
     }
 
-    const blob = await r.blob();
-    const url = window.URL.createObjectURL(blob);
+    const { token: downloadToken } = await tokenResponse.json();
+
+    // Use the token to trigger browser download (no memory loading)
+    const downloadUrl = `${BASE}/files/download?token=${downloadToken}`;
     const a = document.createElement("a");
-    a.href = url;
+    a.href = downloadUrl;
     a.download = filename;
     document.body.appendChild(a);
     a.click();
-    window.URL.revokeObjectURL(url);
     document.body.removeChild(a);
   } catch (e) {
     console.error("Download error:", e);