fix: implement streaming downloads for large files

- Add token-based download system with 5-minute expiry
- Stream files in 8MB chunks on backend (no memory loading)
- Use browser native download with tokens on frontend
- Fixes hang on large files (10GB+) by avoiding memory buffering

claude-code-tracker/collector.py

@@ -52,9 +52,13 @@ async def process_file(file_path: str):
             # No new lines
             return 0
 
+        # Process in batches for large files (max 200 lines at a time)
+        batch_size = 200
+        end_line = min(start_line + batch_size, total_lines)
+
         # Parse new content
-        logger.info(f"Processing {file_path} from line {start_line} to {total_lines}")
+        logger.info(f"Processing {file_path} from line {start_line} to {end_line} (total: {total_lines})")
-        conversation_data = parser.parse_conversation_file(file_path, start_line)
+        conversation_data = parser.parse_conversation_file(file_path, start_line, end_line)
 
         if not conversation_data:
             logger.warning(f"No data extracted from {file_path}")
@@ -87,10 +91,10 @@ async def process_file(file_path: str):
         for tool_call in conversation_data['tool_calls']:
             await db.insert_tool_call(tool_call)
 
-        # Update checkpoint
+        # Update checkpoint (use end_line instead of total_lines for batch processing)
-        await db.update_checkpoint(file_path, total_lines, file_mtime)
+        await db.update_checkpoint(file_path, end_line, file_mtime)
 
-        logger.info(f"Processed {len(conversation_data['messages'])} messages from {file_path}")
+        logger.info(f"Processed {len(conversation_data['messages'])} messages from {file_path} (batch {start_line}-{end_line}/{total_lines})")
         return len(conversation_data['messages'])
 
     except Exception as e:

claude-code-tracker/parser.py

@@ -6,14 +6,16 @@ from typing import Dict, List, Any, Optional
 logger = logging.getLogger(__name__)
 
 
-def parse_jsonl_file(file_path: str, start_line: int = 0) -> List[Dict[str, Any]]:
+def parse_jsonl_file(file_path: str, start_line: int = 0, end_line: int = None) -> List[Dict[str, Any]]:
-    """Parse JSONL file from a specific line number"""
+    """Parse JSONL file from a specific line number to end_line (or EOF if None)"""
     messages = []
     try:
         with open(file_path, 'r', encoding='utf-8') as f:
             for i, line in enumerate(f):
                 if i < start_line:
                     continue
+                if end_line is not None and i >= end_line:
+                    break
                 if not line.strip():
                     continue
                 try:
@@ -169,9 +171,9 @@ def calculate_conversation_stats(messages: List[Dict[str, Any]]) -> Dict[str, An
     return stats
 
 
-def parse_conversation_file(file_path: str, start_line: int = 0) -> Dict[str, Any]:
+def parse_conversation_file(file_path: str, start_line: int = 0, end_line: int = None) -> Dict[str, Any]:
     """Parse a conversation file and extract all data"""
-    raw_messages = parse_jsonl_file(file_path, start_line)
+    raw_messages = parse_jsonl_file(file_path, start_line, end_line)
 
     if not raw_messages:
         return None

dashboard/backend/routers/files.py

@@ -1,11 +1,13 @@
 import logging
 import os
 import re
+import secrets
 import shutil
+import time
 from pathlib import Path
 
-from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
+from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, Query
-from fastapi.responses import FileResponse
+from fastapi.responses import StreamingResponse
 
 from config import VOLUME_ROOT
 from rbac import require_admin
@@ -17,6 +19,10 @@ BASE = Path(VOLUME_ROOT)
 MAX_UPLOAD_SIZE = 100 * 1024 * 1024  # 100 MB
 BLOCKED_DIRS = {"@appstore", "@docker", "@eaDir", "@S2S", "@sharesnap", "@tmp", "docker", "homes", "backups"}
 
+# Download token storage: {token: (path, expiry_timestamp)}
+_download_tokens = {}
+TOKEN_EXPIRY = 300  # 5 minutes
+
 
 _BASE_RESOLVED = str(BASE.resolve())
 
@@ -88,13 +94,65 @@ def browse(path: str = ""):
     return {"path": str(target.relative_to(BASE)), "entries": entries}
 
 
-@router.get("/download")
+@router.post("/download-token")
-def download(path: str):
+def create_download_token(path: str):
+    """Generate a temporary download token for large file downloads."""
     try:
         target = _safe_path(path)
         if not target.exists():
             raise HTTPException(status_code=404, detail="File not found")
-        return FileResponse(target)
+
+        # Clean up expired tokens
+        now = time.time()
+        expired = [t for t, (_, exp) in _download_tokens.items() if exp < now]
+        for t in expired:
+            del _download_tokens[t]
+
+        # Generate new token
+        token = secrets.token_urlsafe(32)
+        _download_tokens[token] = (str(target), now + TOKEN_EXPIRY)
+
+        return {"token": token, "expires_in": TOKEN_EXPIRY}
+    except ValueError:
+        raise HTTPException(status_code=403, detail="Access denied")
+
+
+@router.get("/download")
+def download(path: str = None, token: str = Query(None)):
+    """Download a file. Supports both authenticated access (path param) and token-based access (token param)."""
+    try:
+        if token:
+            # Token-based download (no auth required)
+            if token not in _download_tokens:
+                raise HTTPException(status_code=403, detail="Invalid or expired token")
+
+            file_path, expiry = _download_tokens[token]
+            if time.time() > expiry:
+                del _download_tokens[token]
+                raise HTTPException(status_code=403, detail="Token expired")
+
+            # Consume token after use
+            del _download_tokens[token]
+            target = Path(file_path)
+        else:
+            # Authenticated download
+            if not path:
+                raise HTTPException(status_code=400, detail="Path or token required")
+            target = _safe_path(path)
+
+        if not target.exists():
+            raise HTTPException(status_code=404, detail="File not found")
+
+        def file_iterator():
+            with open(target, "rb") as f:
+                while chunk := f.read(8192 * 1024):  # 8MB chunks
+                    yield chunk
+
+        return StreamingResponse(
+            file_iterator(),
+            media_type="application/octet-stream",
+            headers={"Content-Disposition": f'attachment; filename="{target.name}"'}
+        )
     except ValueError:
         raise HTTPException(status_code=403, detail="Access denied")
 

dashboard/frontend/src/lib/api.js

@@ -202,20 +202,31 @@ export async function download(path, filename) {
   const headers = {};
   if (token) headers["Authorization"] = `Bearer ${token}`;
 
-  const r = await fetch(`${BASE}/files/download?path=${encodeURIComponent(path)}`, {
+  try {
-    method: "GET",
+    // Get a temporary download token
-    headers,
+    const tokenResponse = await fetch(`${BASE}/files/download-token?path=${encodeURIComponent(path)}`, {
-  });
+      method: "POST",
+      headers,
+    });
 
-  if (!r.ok) throw new Error(`HTTP ${r.status}`);
+    if (!tokenResponse.ok) {
+      const errorText = await tokenResponse.text();
+      console.error(`Download token failed: ${tokenResponse.status} - ${errorText}`);
+      throw new Error(`Download failed: ${tokenResponse.status}`);
+    }
 
-  const blob = await r.blob();
+    const { token: downloadToken } = await tokenResponse.json();
-  const url = window.URL.createObjectURL(blob);
+
-  const a = document.createElement("a");
+    // Use the token to trigger browser download (no memory loading)
-  a.href = url;
+    const downloadUrl = `${BASE}/files/download?token=${downloadToken}`;
-  a.download = filename;
+    const a = document.createElement("a");
-  document.body.appendChild(a);
+    a.href = downloadUrl;
-  a.click();
+    a.download = filename;
-  window.URL.revokeObjectURL(url);
+    document.body.appendChild(a);
-  document.body.removeChild(a);
+    a.click();
+    document.body.removeChild(a);
+  } catch (e) {
+    console.error("Download error:", e);
+    throw e;
+  }
 }

dashboard/frontend/src/routes/Files.svelte

@@ -6,6 +6,8 @@
   let currentPath = $state("");
   let loading = $state(true);
   let uploading = $state(false);
+  let downloading = $state(false);
+  let error = $state("");
   let fileInput;
 
   async function browse(path) {
@@ -31,7 +33,16 @@
 
   async function handleDownload(name) {
     const path = currentPath ? `${currentPath}/${name}` : name;
-    await download(path, name);
+    downloading = true;
+    error = "";
+    try {
+      await download(path, name);
+    } catch (e) {
+      error = `Failed to download "${name}": ${e.message}`;
+      console.error("Download failed:", e);
+    } finally {
+      downloading = false;
+    }
   }
 
   async function handleUpload() {
@@ -84,6 +95,23 @@
     </div>
   </div>
 
+  <!-- Error message -->
+  {#if error}
+    <div class="bg-rose-50 dark:bg-rose-900/20 border border-rose-200 dark:border-rose-800 rounded-lg px-4 py-3 flex items-start gap-3">
+      <svg class="w-5 h-5 text-rose-500 shrink-0 mt-0.5" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
+        <path stroke-linecap="round" stroke-linejoin="round" d="M12 8v4m0 4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
+      </svg>
+      <div class="flex-1">
+        <p class="text-sm text-rose-700 dark:text-rose-300">{error}</p>
+      </div>
+      <button onclick={() => error = ""} class="text-rose-400 hover:text-rose-600 transition-colors">
+        <svg class="w-4 h-4" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
+          <path stroke-linecap="round" stroke-linejoin="round" d="M6 18L18 6M6 6l12 12" />
+        </svg>
+      </button>
+    </div>
+  {/if}
+
   <!-- Breadcrumbs -->
   <div class="flex items-center gap-1 text-sm flex-wrap">
     <button class="text-primary-600 hover:text-primary-700 font-medium transition-colors" onclick={() => browse("")}>volume1</button>
@@ -137,7 +165,9 @@
           <span class="text-xs text-surface-400 text-right">{e.is_dir ? "—" : formatSize(e.size)}</span>
           <div class="flex items-center justify-end gap-1 opacity-100 md:opacity-0 md:group-hover:opacity-100 transition-opacity">
             {#if !e.is_dir}
-              <button onclick={() => handleDownload(e.name)} class="text-[11px] text-primary-500 hover:text-primary-700 font-medium transition-colors">DL</button>
+              <button onclick={() => handleDownload(e.name)} class="text-[11px] text-primary-500 hover:text-primary-700 font-medium transition-colors disabled:opacity-50" disabled={downloading}>
+                {downloading ? "..." : "DL"}
+              </button>
             {/if}
             <button class="text-[11px] text-rose-400 hover:text-rose-600 font-medium transition-colors" onclick={() => remove(e.name)}>Del</button>
           </div>