sprint: Media Ops — shuttle status endpoint, graceful offline mode, 18 backend tests, shuttle detail in pipeline card, error states with retry, 13 frontend vitest tests

dashboard/backend/routers/media_mgmt.py

@@ -1,6 +1,9 @@
 import json
 import logging
 import os
+import re
+from datetime import datetime
+
 import asyncssh
 from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks, Query
 import config
@@ -18,16 +21,22 @@ PHONE_INGEST_SCRIPT = "/volume1/homes/zjgump/media-mgmt/transfer/phone-to-nas.sh
 
 LOG_DIR = "/volume1/homes/zjgump/media-mgmt/logs"
 
+UNAVAILABLE_RESPONSE = {"status": "unavailable", "error": "NAS unreachable"}
+
+
 async def run_host_command(cmd: str) -> tuple[int, str, str]:
-    """Execute a command on the NAS host via SSH."""
+    """Execute a command on the NAS host via SSH.
+
+    Returns (exit_code, stdout, stderr). On SSH connection failure,
+    exit_code=999 and stderr contains the error message.
+    """
     try:
-        # Prepend PATH and NAS_HOST variables for host execution
         full_cmd = f'PATH="/volume1/homes/zjgump/bin:$PATH" NAS_HOST="naslan" {cmd}'
         async with asyncssh.connect(
             config.SSH_HOST,
             username=config.SSH_USER,
             client_keys=[config.SSH_KEY_PATH],
-            known_hosts=None
+            known_hosts=None,
         ) as conn:
             result = await conn.run(full_cmd)
             return result.exit_status, result.stdout, result.stderr
@@ -35,78 +44,148 @@ async def run_host_command(cmd: str) -> tuple[int, str, str]:
         logger.error(f"SSH command failed ({cmd}): {e}")
         return 999, "", str(e)
 
+
 @router.get("/health")
 async def get_health():
-    """Runs health-check.sh and returns the exit status and stdout/stderr."""
+    """Runs health-check.sh and returns structured status."""
     code, stdout, stderr = await run_host_command(f"bash {HEALTH_CHECK_SCRIPT}")
+    if code == 999:
+        return UNAVAILABLE_RESPONSE
     return {
         "status": "ok" if code == 0 else "warning" if code == 1 else "error",
         "exit_code": code,
         "stdout": stdout,
-        "stderr": stderr
+        "stderr": stderr,
     }
 
+
 @router.get("/storage")
 async def get_storage():
     """Runs storage-report.sh -j and returns the parsed JSON storage report."""
     code, stdout, stderr = await run_host_command(f"bash {STORAGE_REPORT_SCRIPT} -j")
+    if code == 999:
+        return UNAVAILABLE_RESPONSE
     if code != 0:
-        raise HTTPException(status_code=500, detail=f"Storage report failed: {stderr}")
+        return {"error": "Storage report failed", "stderr": stderr}
     try:
         return json.loads(stdout)
     except json.JSONDecodeError:
-        return {
-            "error": "Failed to parse JSON from storage report",
-            "raw_output": stdout,
-            "stderr": stderr
-        }
+        return {"error": "Failed to parse JSON from storage report", "raw_output": stdout}
+
+
+@router.get("/shuttle/status")
+async def get_shuttle_status():
+    """Shuttle pipeline status: queue depth, last transfer time, recent errors.
+
+    Reads the shuttle log file to extract pipeline state.
+    """
+    log_path = f"{LOG_DIR}/shuttle.log"
+    code, stdout, stderr = await run_host_command(f"cat {log_path} 2>/dev/null || echo 'NO_LOG'")
+    if code == 999:
+        return UNAVAILABLE_RESPONSE
+
+    result = {
+        "running": False,
+        "queue_depth": 0,
+        "last_transfer": None,
+        "last_transfer_size": None,
+        "recent_errors": [],
+        "current_stage": None,
+    }
+
+    content = stdout or ""
+    if content.strip() == "NO_LOG":
+        return result
+
+    # Check if shuttle is currently running via pgrep
+    running_code, running_out, _ = await run_host_command(f"pgrep -f {SHUTTLE_SCRIPT}")
+    result["running"] = running_code == 0
+    if running_code == 0:
+        result["current_stage"] = "active"
+        result["pid"] = running_out.strip()
+
+    # Parse log for last transfer, errors, current stage
+    lines = content.strip().split("\n")
+    error_lines = []
+    transfer_lines = []
+
+    for line in lines:
+        if "❌" in line or "error" in line.lower() or "failed" in line.lower():
+            error_lines.append(line.strip())
+        if "copied" in line.lower() or "transferred" in line.lower() or "rsync" in line.lower():
+            transfer_lines.append(line.strip())
+        if "Stage" in line and ":" in line:
+            result["current_stage"] = line.strip()
+
+    if error_lines:
+        result["recent_errors"] = error_lines[-10:]
+
+    if transfer_lines:
+        last = transfer_lines[-1]
+        result["last_transfer"] = last
+        # Try to extract size from rsync-like output
+        size_match = re.search(r"(\d+\.?\d*)\s*(K|M|G)B", last)
+        if size_match:
+            result["last_transfer_size"] = size_match.group(0)
+
+    # Count queued source directories
+    queue_matches = [l for l in lines if "--source" in l or "--stage" in l]
+    result["queue_depth"] = len(queue_matches)
+
+    return result
+
 
 @router.post("/maintenance/scan")
 async def run_scan(background_tasks: BackgroundTasks, library: str = Query(None)):
-    """Triggers scan-library.sh in the background and writes stdout/stderr to scan.log."""
+    """Triggers scan-library.sh in the background."""
     cmd = f"bash {SCAN_LIBRARY_SCRIPT}"
     if library:
         cmd += f" {library}"
-    
+
     log_file = f"{LOG_DIR}/scan.log"
     run_cmd = f"mkdir -p {LOG_DIR} && echo '=== Scan Started at $(date) ===' > {log_file} && nohup {cmd} >> {log_file} 2>&1 &"
-    
+
     code, stdout, stderr = await run_host_command(run_cmd)
+    if code == 999:
+        return UNAVAILABLE_RESPONSE
     if code != 0:
-        raise HTTPException(status_code=500, detail=f"Failed to start scan: {stderr}")
-    
+        return {"error": "Failed to start scan", "stderr": stderr}
+
     return {"message": "Scan started in background", "log_file": log_file}
 
+
 @router.post("/maintenance/requeue")
 async def run_requeue(background_tasks: BackgroundTasks):
-    """Triggers thumbnail-clinic.sh -r in the background and writes stdout/stderr to requeue.log."""
+    """Triggers thumbnail-clinic.sh -r in the background."""
     cmd = f"bash {THUMBNAIL_CLINIC_SCRIPT} -r"
     log_file = f"{LOG_DIR}/requeue.log"
-    
     run_cmd = f"mkdir -p {LOG_DIR} && echo '=== Requeue Started at $(date) ===' > {log_file} && nohup {cmd} >> {log_file} 2>&1 &"
-    
+
     code, stdout, stderr = await run_host_command(run_cmd)
+    if code == 999:
+        return UNAVAILABLE_RESPONSE
     if code != 0:
-        raise HTTPException(status_code=500, detail=f"Failed to start thumbnail requeue: {stderr}")
-    
+        return {"error": "Failed to start thumbnail requeue", "stderr": stderr}
+
     return {"message": "Thumbnail requeue started in background", "log_file": log_file}
 
+
 @router.get("/logs/{task_name}")
 async def get_logs(task_name: str):
-    """Reads logs for a task from /volume1/homes/zjgump/media-mgmt/logs/{task_name}.log."""
+    """Reads logs for a task from the media-mgmt logs directory."""
     if task_name not in ["scan", "requeue", "shuttle", "phone-ingest"]:
         raise HTTPException(status_code=400, detail="Invalid task name")
-    
-    log_path = f"/volume1/homes/zjgump/media-mgmt/logs/{task_name}.log"
-    if not os.path.exists(log_path):
+
+    log_path = f"{LOG_DIR}/{task_name}.log"
+    code, stdout, stderr = await run_host_command(f"cat {log_path} 2>/dev/null || echo 'NO_LOG'")
+    if code == 999:
+        return UNAVAILABLE_RESPONSE
+
+    if stdout.strip() == "NO_LOG":
         return {"content": "No log file found."}
-    
-    try:
-        with open(log_path, "r") as f:
-            content = f.readlines()
-            return {"content": "".join(content[-200:])}
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Failed to read log file: {e}")
+    lines = stdout.strip().split("\n")
+    return {"content": "\n".join(lines[-200:])}
+
 
 @router.get("/status")
 async def get_active_status():
@@ -115,14 +194,16 @@ async def get_active_status():
         "scan": SCAN_LIBRARY_SCRIPT,
         "requeue": THUMBNAIL_CLINIC_SCRIPT,
         "shuttle": SHUTTLE_SCRIPT,
-        "phone-ingest": PHONE_INGEST_SCRIPT
+        "phone-ingest": PHONE_INGEST_SCRIPT,
     }
-    
+
     status = {}
     for name, script_path in processes.items():
         code, stdout, stderr = await run_host_command(f"pgrep -f {script_path}")
+        if code == 999:
+            return UNAVAILABLE_RESPONSE
         status[name] = {
             "running": code == 0,
-            "pid": stdout.strip() if code == 0 else None
+            "pid": stdout.strip() if code == 0 else None,
         }
     return status

dashboard/backend/tests/integration/test_media_mgmt.py

@@ -1,8 +1,9 @@
 import pytest
 from unittest.mock import patch, AsyncMock
 
+
 @pytest.mark.asyncio
-async def test_get_health(test_app, admin_headers):
+async def test_get_health_ok(test_app, admin_headers):
     with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
         mock_run.return_value = (0, "All checks passed\n", "")
         response = test_app.get("/api/media/health", headers=admin_headers)
@@ -12,16 +13,145 @@ async def test_get_health(test_app, admin_headers):
         assert data["exit_code"] == 0
         assert "All checks passed" in data["stdout"]
 
+
+@pytest.mark.asyncio
+async def test_get_health_warning(test_app, admin_headers):
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        mock_run.return_value = (1, "Disk >85%\n", "")
+        response = test_app.get("/api/media/health", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "warning"
+
+
+@pytest.mark.asyncio
+async def test_get_health_error(test_app, admin_headers):
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        mock_run.return_value = (2, "", "Immich API unreachable")
+        response = test_app.get("/api/media/health", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "error"
+
+
+@pytest.mark.asyncio
+async def test_get_health_unavailable(test_app, admin_headers):
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        mock_run.return_value = (999, "", "SSH connection refused")
+        response = test_app.get("/api/media/health", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "unavailable"
+        assert "NAS unreachable" in data["error"]
+
+
 @pytest.mark.asyncio
 async def test_get_storage(test_app, admin_headers):
     with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
-        mock_run.return_value = (0, '{"timestamp": "2026-07-11T05:37:57Z", "volume": {"path": "/volume1", "total": "14T", "used": "12T", "available": "2.2T", "used_pct": 85}}', "")
+        mock_run.return_value = (
+            0,
+            '{"timestamp": "2026-07-11T05:37:57Z", "volume": {"path": "/volume1", "total": "14T", "used": "12T", "available": "2.2T", "used_pct": 85}}',
+            "",
+        )
         response = test_app.get("/api/media/storage", headers=admin_headers)
         assert response.status_code == 200
         data = response.json()
         assert "volume" in data
         assert data["volume"]["total"] == "14T"
 
+
+@pytest.mark.asyncio
+async def test_get_storage_unavailable(test_app, admin_headers):
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        mock_run.return_value = (999, "", "SSH timeout")
+        response = test_app.get("/api/media/storage", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "unavailable"
+
+
+@pytest.mark.asyncio
+async def test_get_storage_json_parse_error(test_app, admin_headers):
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        mock_run.return_value = (0, "not valid json", "")
+        response = test_app.get("/api/media/storage", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert "error" in data
+        assert "Failed to parse JSON" in data["error"]
+
+
+@pytest.mark.asyncio
+async def test_get_shuttle_status_idle(test_app, admin_headers):
+    """Shuttle not running, no log file."""
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        # First call: cat log, Second call: pgrep -f
+        mock_run.side_effect = [
+            (0, "NO_LOG", ""),  # cat log
+            (1, "", ""),  # pgrep -f (not running)
+        ]
+        response = test_app.get("/api/media/shuttle/status", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert data["running"] is False
+        assert data["queue_depth"] == 0
+
+
+@pytest.mark.asyncio
+async def test_get_shuttle_status_running(test_app, admin_headers):
+    """Shuttle running with log content."""
+    log_content = """=== Shuttle Started at 2026-07-11 10:00 ===
+--source /Volumes/SD_CARD --dest /Volumes/SSD
+Stage 1: copy
+[10:01] rsync: 1.2GB copied from SD_CARD to SSD
+Stage 2: verify
+[10:05] ✅ Verification passed — all files match
+[10:06] --stage ship --source /Volumes/SSD --dest naslan:/volume1/video
+"""
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        mock_run.side_effect = [
+            (0, log_content, ""),  # cat log
+            (0, "12345\n", ""),  # pgrep -f (running)
+        ]
+        response = test_app.get("/api/media/shuttle/status", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert data["running"] is True
+        assert data["pid"] == "12345"
+        assert data["queue_depth"] == 2  # --source entries in log
+        assert data["last_transfer"] is not None
+        assert "1.2GB" in data.get("last_transfer_size", "")
+
+
+@pytest.mark.asyncio
+async def test_get_shuttle_status_with_errors(test_app, admin_headers):
+    """Shuttle log has error lines."""
+    log_content = """[10:01] rsync: OK
+[10:02] ❌ rsync: failed to transfer file XYZ
+[10:03] rsync: 500MB copied
+[10:04] ❌ connection reset by peer
+"""
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        mock_run.side_effect = [
+            (0, log_content, ""),  # cat log
+            (1, "", ""),  # pgrep -f (not running)
+        ]
+        response = test_app.get("/api/media/shuttle/status", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert len(data["recent_errors"]) == 2
+
+
+@pytest.mark.asyncio
+async def test_get_shuttle_status_unavailable(test_app, admin_headers):
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        mock_run.return_value = (999, "", "SSH timeout")
+        response = test_app.get("/api/media/shuttle/status", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "unavailable"
+
+
 @pytest.mark.asyncio
 async def test_run_scan(test_app, admin_headers):
     with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
@@ -33,6 +163,17 @@ async def test_run_scan(test_app, admin_headers):
         mock_run.assert_called_once()
         assert "scan-library.sh DJI-Action6" in mock_run.call_args[0][0]
 
+
+@pytest.mark.asyncio
+async def test_run_scan_unavailable(test_app, admin_headers):
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        mock_run.return_value = (999, "", "SSH timeout")
+        response = test_app.post("/api/media/maintenance/scan", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "unavailable"
+
+
 @pytest.mark.asyncio
 async def test_get_active_status(test_app, admin_headers):
     with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
@@ -47,3 +188,41 @@ async def test_get_active_status(test_app, admin_headers):
         assert data["scan"]["running"] is True
         assert data["scan"]["pid"] == "1234"
         assert data["requeue"]["running"] is False
+
+
+@pytest.mark.asyncio
+async def test_get_active_status_unavailable(test_app, admin_headers):
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        mock_run.return_value = (999, "", "SSH timeout")
+        response = test_app.get("/api/media/status", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "unavailable"
+
+
+@pytest.mark.asyncio
+async def test_get_logs_shuttle(test_app, admin_headers):
+    """Fetch shuttle logs successfully."""
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        mock_run.return_value = (0, "line1\nline2\nline3\n", "")
+        response = test_app.get("/api/media/logs/shuttle", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert "line1" in data["content"]
+
+
+@pytest.mark.asyncio
+async def test_get_logs_invalid_task(test_app, admin_headers):
+    """Invalid task name returns 400."""
+    response = test_app.get("/api/media/logs/invalid-task", headers=admin_headers)
+    assert response.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_get_logs_unavailable(test_app, admin_headers):
+    with patch("routers.media_mgmt.run_host_command", new_callable=AsyncMock) as mock_run:
+        mock_run.return_value = (999, "", "SSH timeout")
+        response = test_app.get("/api/media/logs/scan", headers=admin_headers)
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "unavailable"