sprint: Media Ops — shuttle status endpoint, graceful offline mode, 18 backend tests, shuttle detail in pipeline card, error states with retry, 13 frontend vitest tests

dashboard/backend/routers/media_mgmt.py

@@ -1,6 +1,9 @@
 import json
 import logging
 import os
+import re
+from datetime import datetime
+
 import asyncssh
 from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks, Query
 import config
@@ -18,16 +21,22 @@ PHONE_INGEST_SCRIPT = "/volume1/homes/zjgump/media-mgmt/transfer/phone-to-nas.sh
 
 LOG_DIR = "/volume1/homes/zjgump/media-mgmt/logs"
 
+UNAVAILABLE_RESPONSE = {"status": "unavailable", "error": "NAS unreachable"}
+
+
 async def run_host_command(cmd: str) -> tuple[int, str, str]:
-    """Execute a command on the NAS host via SSH."""
+    """Execute a command on the NAS host via SSH.
+
+    Returns (exit_code, stdout, stderr). On SSH connection failure,
+    exit_code=999 and stderr contains the error message.
+    """
     try:
-        # Prepend PATH and NAS_HOST variables for host execution
         full_cmd = f'PATH="/volume1/homes/zjgump/bin:$PATH" NAS_HOST="naslan" {cmd}'
         async with asyncssh.connect(
             config.SSH_HOST,
             username=config.SSH_USER,
             client_keys=[config.SSH_KEY_PATH],
-            known_hosts=None
+            known_hosts=None,
         ) as conn:
             result = await conn.run(full_cmd)
             return result.exit_status, result.stdout, result.stderr
@@ -35,78 +44,148 @@ async def run_host_command(cmd: str) -> tuple[int, str, str]:
         logger.error(f"SSH command failed ({cmd}): {e}")
         return 999, "", str(e)
 
+
 @router.get("/health")
 async def get_health():
-    """Runs health-check.sh and returns the exit status and stdout/stderr."""
+    """Runs health-check.sh and returns structured status."""
     code, stdout, stderr = await run_host_command(f"bash {HEALTH_CHECK_SCRIPT}")
+    if code == 999:
+        return UNAVAILABLE_RESPONSE
     return {
         "status": "ok" if code == 0 else "warning" if code == 1 else "error",
         "exit_code": code,
         "stdout": stdout,
-        "stderr": stderr
+        "stderr": stderr,
     }
 
+
 @router.get("/storage")
 async def get_storage():
     """Runs storage-report.sh -j and returns the parsed JSON storage report."""
     code, stdout, stderr = await run_host_command(f"bash {STORAGE_REPORT_SCRIPT} -j")
+    if code == 999:
+        return UNAVAILABLE_RESPONSE
     if code != 0:
-        raise HTTPException(status_code=500, detail=f"Storage report failed: {stderr}")
+        return {"error": "Storage report failed", "stderr": stderr}
     try:
         return json.loads(stdout)
     except json.JSONDecodeError:
-        return {
-            "error": "Failed to parse JSON from storage report",
-            "raw_output": stdout,
-            "stderr": stderr
-        }
+        return {"error": "Failed to parse JSON from storage report", "raw_output": stdout}
+
+
+@router.get("/shuttle/status")
+async def get_shuttle_status():
+    """Shuttle pipeline status: queue depth, last transfer time, recent errors.
+
+    Reads the shuttle log file to extract pipeline state.
+    """
+    log_path = f"{LOG_DIR}/shuttle.log"
+    code, stdout, stderr = await run_host_command(f"cat {log_path} 2>/dev/null || echo 'NO_LOG'")
+    if code == 999:
+        return UNAVAILABLE_RESPONSE
+
+    result = {
+        "running": False,
+        "queue_depth": 0,
+        "last_transfer": None,
+        "last_transfer_size": None,
+        "recent_errors": [],
+        "current_stage": None,
+    }
+
+    content = stdout or ""
+    if content.strip() == "NO_LOG":
+        return result
+
+    # Check if shuttle is currently running via pgrep
+    running_code, running_out, _ = await run_host_command(f"pgrep -f {SHUTTLE_SCRIPT}")
+    result["running"] = running_code == 0
+    if running_code == 0:
+        result["current_stage"] = "active"
+        result["pid"] = running_out.strip()
+
+    # Parse log for last transfer, errors, current stage
+    lines = content.strip().split("\n")
+    error_lines = []
+    transfer_lines = []
+
+    for line in lines:
+        if "❌" in line or "error" in line.lower() or "failed" in line.lower():
+            error_lines.append(line.strip())
+        if "copied" in line.lower() or "transferred" in line.lower() or "rsync" in line.lower():
+            transfer_lines.append(line.strip())
+        if "Stage" in line and ":" in line:
+            result["current_stage"] = line.strip()
+
+    if error_lines:
+        result["recent_errors"] = error_lines[-10:]
+
+    if transfer_lines:
+        last = transfer_lines[-1]
+        result["last_transfer"] = last
+        # Try to extract size from rsync-like output
+        size_match = re.search(r"(\d+\.?\d*)\s*(K|M|G)B", last)
+        if size_match:
+            result["last_transfer_size"] = size_match.group(0)
+
+    # Count queued source directories
+    queue_matches = [l for l in lines if "--source" in l or "--stage" in l]
+    result["queue_depth"] = len(queue_matches)
+
+    return result
+
 
 @router.post("/maintenance/scan")
 async def run_scan(background_tasks: BackgroundTasks, library: str = Query(None)):
-    """Triggers scan-library.sh in the background and writes stdout/stderr to scan.log."""
+    """Triggers scan-library.sh in the background."""
     cmd = f"bash {SCAN_LIBRARY_SCRIPT}"
     if library:
         cmd += f" {library}"
-    
+
     log_file = f"{LOG_DIR}/scan.log"
     run_cmd = f"mkdir -p {LOG_DIR} && echo '=== Scan Started at $(date) ===' > {log_file} && nohup {cmd} >> {log_file} 2>&1 &"
-    
+
     code, stdout, stderr = await run_host_command(run_cmd)
+    if code == 999:
+        return UNAVAILABLE_RESPONSE
     if code != 0:
-        raise HTTPException(status_code=500, detail=f"Failed to start scan: {stderr}")
-    
+        return {"error": "Failed to start scan", "stderr": stderr}
+
     return {"message": "Scan started in background", "log_file": log_file}
 
+
 @router.post("/maintenance/requeue")
 async def run_requeue(background_tasks: BackgroundTasks):
-    """Triggers thumbnail-clinic.sh -r in the background and writes stdout/stderr to requeue.log."""
+    """Triggers thumbnail-clinic.sh -r in the background."""
     cmd = f"bash {THUMBNAIL_CLINIC_SCRIPT} -r"
     log_file = f"{LOG_DIR}/requeue.log"
-    
     run_cmd = f"mkdir -p {LOG_DIR} && echo '=== Requeue Started at $(date) ===' > {log_file} && nohup {cmd} >> {log_file} 2>&1 &"
-    
+
     code, stdout, stderr = await run_host_command(run_cmd)
+    if code == 999:
+        return UNAVAILABLE_RESPONSE
     if code != 0:
-        raise HTTPException(status_code=500, detail=f"Failed to start thumbnail requeue: {stderr}")
-    
+        return {"error": "Failed to start thumbnail requeue", "stderr": stderr}
+
     return {"message": "Thumbnail requeue started in background", "log_file": log_file}
 
+
 @router.get("/logs/{task_name}")
 async def get_logs(task_name: str):
-    """Reads logs for a task from /volume1/homes/zjgump/media-mgmt/logs/{task_name}.log."""
+    """Reads logs for a task from the media-mgmt logs directory."""
     if task_name not in ["scan", "requeue", "shuttle", "phone-ingest"]:
         raise HTTPException(status_code=400, detail="Invalid task name")
-    
-    log_path = f"/volume1/homes/zjgump/media-mgmt/logs/{task_name}.log"
-    if not os.path.exists(log_path):
+
+    log_path = f"{LOG_DIR}/{task_name}.log"
+    code, stdout, stderr = await run_host_command(f"cat {log_path} 2>/dev/null || echo 'NO_LOG'")
+    if code == 999:
+        return UNAVAILABLE_RESPONSE
+
+    if stdout.strip() == "NO_LOG":
         return {"content": "No log file found."}
-    
-    try:
-        with open(log_path, "r") as f:
-            content = f.readlines()
-            return {"content": "".join(content[-200:])}
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Failed to read log file: {e}")
+    lines = stdout.strip().split("\n")
+    return {"content": "\n".join(lines[-200:])}
+
 
 @router.get("/status")
 async def get_active_status():
@@ -115,14 +194,16 @@ async def get_active_status():
         "scan": SCAN_LIBRARY_SCRIPT,
         "requeue": THUMBNAIL_CLINIC_SCRIPT,
         "shuttle": SHUTTLE_SCRIPT,
-        "phone-ingest": PHONE_INGEST_SCRIPT
+        "phone-ingest": PHONE_INGEST_SCRIPT,
     }
-    
+
     status = {}
     for name, script_path in processes.items():
         code, stdout, stderr = await run_host_command(f"pgrep -f {script_path}")
+        if code == 999:
+            return UNAVAILABLE_RESPONSE
         status[name] = {
             "running": code == 0,
-            "pid": stdout.strip() if code == 0 else None
+            "pid": stdout.strip() if code == 0 else None,
         }
     return status