sprint1: agent queue worker, proxy support, auto-mark-on-navigate-away, test fixes

backend/agent_worker.py

@@ -0,0 +1,166 @@
+"""
+Agent Queue Worker — polls agent_queue for pending items and
+processes them against the local Qwen27B API (:8080).
+"""
+import asyncio
+import json
+import logging
+import os
+import aiosqlite
+import httpx
+
+import db
+
+log = logging.getLogger(__name__)
+
+QWEN_BASE_URL = os.environ.get("QWEN_BASE_URL", "http://localhost:8080")
+QWEN_MODEL = os.environ.get("QWEN_MODEL", "current")
+POLL_INTERVAL = int(os.environ.get("AGENT_POLL_INTERVAL", "30"))
+MAX_CONCURRENT = int(os.environ.get("AGENT_MAX_CONCURRENT", "3"))
+SYSTEM_PROMPT = os.environ.get(
+    "AGENT_SYSTEM_PROMPT",
+    "You are a helpful AI assistant. The user has watched a YouTube video and "
+    "left you instructions. Read the video info below and respond to their "
+    "prompt. Be concise and actionable."
+)
+
+semaphore = asyncio.Semaphore(MAX_CONCURRENT)
+
+
+async def _call_qwen(prompt: str, video_title: str, video_context: str) -> str:
+    """Call the local Qwen27B API with video context + user prompt."""
+    messages = [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {
+            "role": "user",
+            "content": (
+                f"## Video Title\n{video_title}\n\n"
+                f"## Video Context\n{video_context}\n\n"
+                f"## User's Request\n{prompt}"
+            ),
+        },
+    ]
+
+    http_proxy = os.environ.get("HTTP_PROXY") or os.environ.get("http_proxy")
+    https_proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy")
+    proxies = http_proxy or https_proxy
+
+    async with httpx.AsyncClient(proxy=proxies, timeout=180) as client:
+        resp = await client.post(
+            f"{QWEN_BASE_URL}/v1/chat/completions",
+            json={
+                "model": QWEN_MODEL,
+                "messages": messages,
+                "max_tokens": 2048,
+                "temperature": 0.3,
+            },
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        return data["choices"][0]["message"]["content"]
+
+
+async def _process_item(item: dict) -> None:
+    """Process a single agent_queue item."""
+    video_id = item["video_id"]
+    user_prompt = item["user_prompt"]
+
+    async with aiosqlite.connect(db.DB_PATH) as conn:
+        conn.row_factory = aiosqlite.Row
+
+        # Mark as processing
+        await conn.execute(
+            "UPDATE agent_queue SET status = 'processing' WHERE id = ?",
+            (item["id"],),
+        )
+        await conn.commit()
+
+        # Get video info
+        cursor = await conn.execute(
+            "SELECT title, channel_name, summary, transcript FROM videos WHERE video_id = ?",
+            (video_id,),
+        )
+        video = await cursor.fetchone()
+
+    if not video:
+        async with aiosqlite.connect(db.DB_PATH) as conn:
+            await conn.execute(
+                "UPDATE agent_queue SET status = 'failed', result = 'Video not found in DB' WHERE id = ?",
+                (item["id"],),
+            )
+            await conn.commit()
+        return
+
+    # Build video context from whatever we have
+    context_parts = []
+    if video["channel_name"]:
+        context_parts.append(f"Channel: {video['channel_name']}")
+    if video["summary"] and not video["summary"].startswith("📺 Short clip"):
+        context_parts.append(f"Summary:\n{video['summary']}")
+    if video["transcript"] and video["transcript"] != "No transcript available":
+        context_parts.append(f"Transcript (first 4000 chars):\n{video['transcript'][:4000]}")
+    video_context = "\n\n".join(context_parts) if context_parts else "No additional context available."
+
+    try:
+        result = await _call_qwen(user_prompt, video["title"], video_context)
+        status = "done"
+    except Exception as e:
+        log.warning(f"Qwen API call failed for item {item['id']}: {e}")
+        result = f"Error: {e}"
+        status = "failed"
+
+    async with aiosqlite.connect(db.DB_PATH) as conn:
+        await conn.execute(
+            "UPDATE agent_queue SET status = ?, result = ?, processed_at = CURRENT_TIMESTAMP WHERE id = ?",
+            (status, result, item["id"]),
+        )
+        await conn.commit()
+
+
+async def process_queue_loop() -> None:
+    """Main polling loop — runs forever."""
+    log.info(
+        f"Agent worker started: Qwen at {QWEN_BASE_URL}, "
+        f"poll every {POLL_INTERVAL}s, max {MAX_CONCURRENT} concurrent"
+    )
+
+    while True:
+        try:
+            async with aiosqlite.connect(db.DB_PATH) as conn:
+                conn.row_factory = aiosqlite.Row
+                cursor = await conn.execute(
+                    "SELECT id, video_id, user_prompt FROM agent_queue WHERE status = 'pending' ORDER BY created_at ASC LIMIT ?",
+                    (MAX_CONCURRENT,),
+                )
+                items = await cursor.fetchall()
+
+            if items:
+                log.info(f"Agent queue: processing {len(items)} item(s)")
+                tasks = [_process_item(dict(item)) for item in items]
+                await asyncio.gather(*tasks)
+
+        except Exception as e:
+            log.error(f"Agent worker loop error: {e}")
+
+        await asyncio.sleep(POLL_INTERVAL)
+
+
+async def start_worker():
+    """Entry point — called from main.py startup."""
+    task = asyncio.create_task(process_queue_loop())
+    log.info("Agent queue worker scheduled")
+    return task
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+    )
+
+    async def main():
+        from db import init_db
+        await init_db()
+        log.info("Agent worker running in foreground. Ctrl+C to stop.")
+        await process_queue_loop()
+
+    asyncio.run(main())

backend/db.py

@@ -39,6 +39,7 @@ async def init_db():
                 status TEXT DEFAULT 'pending',
                 created_at TEXT DEFAULT CURRENT_TIMESTAMP,
                 processed_at TEXT,
+                result TEXT,
                 FOREIGN KEY (video_id) REFERENCES videos(video_id)
             )
         """)

backend/fetcher.py

@@ -99,7 +99,11 @@ async def fetch_subscriptions():
 
     # Step 3: Fetch RSS from slice channels (free, parallel)
     rss_videos = {}
-    async with httpx.AsyncClient(timeout=20) as client:
+    async with httpx.AsyncClient(
+        proxy=os.environ.get("HTTP_PROXY") or os.environ.get("http_proxy") or
+              os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy") or None,
+        timeout=20
+    ) as client:
         tasks = [_fetch_rss(client, cid) for cid in slice_ids]
         results = await asyncio.gather(*tasks)
         for cid, entries in zip(slice_ids, results):

backend/main.py

@@ -1,7 +1,9 @@
 import os
 import json
 import logging
-from fastapi import FastAPI, BackgroundTasks, Depends, HTTPException, Query
+from fastapi import FastAPI, BackgroundTasks, Depends, HTTPException, Query, Response
+from fastapi import Request
+from fastapi.responses import JSONResponse
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel
@@ -11,6 +13,7 @@ import aiosqlite
 from db import init_db, get_db
 from fetcher import fetch_subscriptions
 from summarizer import summarize_video, summarize_all_pending
+from agent_worker import start_worker
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 log = logging.getLogger(__name__)
@@ -23,8 +26,25 @@ app.add_middleware(
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
+
 )
 
+# API token validation for write operations
+API_TOKEN=os.environ.get('API_TOKEN', '')
+
+async def validate_api_token(request: Request, call_next):
+    path = request.url.path
+    method = request.method
+    # Only check auth for write operations on /api/*
+    if path.startswith("/api/") and method in ("POST", "PATCH", "DELETE") and API_TOKEN:
+        auth_header = request.headers.get("Authorization", "")
+        if not auth_header.startswith("Bearer ") or auth_header[7:] != API_TOKEN:
+            return JSONResponse(status_code=401, content={"detail": "Unauthorized"})
+    return await call_next(request)
+
+app.middleware("http")(validate_api_token)
+
+
 class StatusUpdate(BaseModel):
     status: str
 
@@ -45,11 +65,21 @@ class AgentQueueUpdate(BaseModel):
 @app.on_event("startup")
 async def startup():
     await init_db()
+    if os.environ.get("AGENT_ENABLED", "").lower() == "true":
+        await start_worker()
+        log.info("Agent queue worker enabled")
+    else:
+        log.info("Agent queue worker disabled (set AGENT_ENABLED=true to enable)")
 
 @app.get("/api/health")
 async def health():
     return {"status": "ok"}
 
+@app.get("/api/config")
+async def api_config():
+    """Return API config without exposing the token."""
+    return {"auth_enabled": bool(API_TOKEN)}
+
 @app.get("/auto_categories.json")
 async def auto_categories():
     path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "frontend", "src", "auto_categories.json")

backend/summarizer.py

@@ -9,6 +9,11 @@ from db import DB_PATH
 
 log = logging.getLogger(__name__)
 
+def proxy_from_env():
+    """Return proxy URL from HTTP_PROXY/HTTPS_PROXY env vars, or None."""
+    return os.environ.get("HTTP_PROXY") or os.environ.get("http_proxy") or \
+           os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy") or None
+
 # Fallback pattern to retrieve the correct bytecatcode credentials
 API_KEY = os.environ.get("DEEPSEEK_API_KEY", os.environ.get("OPENAI_API_KEY", ""))
 BASE_URL = os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com")
@@ -108,7 +113,7 @@ async def summarize_video(video_id: str) -> str | None:
         ]
     }
 
-    async with httpx.AsyncClient(timeout=120) as client:
+    async with httpx.AsyncClient(proxy=proxy_from_env(), timeout=120) as client:
         try:
             resp = await client.post(
                 f"{BASE_URL}/v1/chat/completions",

backend/tests/test_agent_worker.py

@@ -0,0 +1,112 @@
+"""Tests for agent_worker.py — mocks Qwen API."""
+import pytest
+import os
+import json
+import aiosqlite
+import httpx
+from unittest.mock import patch, AsyncMock, MagicMock
+
+
+@pytest.mark.asyncio
+async def test_agent_worker_polls_and_processes(test_db, monkeypatch):
+    """Worker picks up a pending item, calls Qwen, marks it done."""
+    from agent_worker import _process_item
+
+    # Seed a pending queue item + video
+    await test_db.execute("""
+        INSERT INTO videos (video_id, title, channel_name, url, summary, status)
+        VALUES ('ag_vid', 'Test Video', 'Test Channel', 'https://youtube.com/watch?v=ag_vid',
+                '**Summary**\n- Point 1\n- Point 2\nVerdict.', 'pending')
+    """)
+    await test_db.execute("""
+        INSERT INTO agent_queue (video_id, user_prompt, status)
+        VALUES ('ag_vid', 'Save this as a skill', 'pending')
+    """)
+    await test_db.commit()
+
+    # Get the item
+    cursor = await test_db.execute(
+        "SELECT id, video_id, user_prompt FROM agent_queue WHERE status = 'pending'"
+    )
+    item = dict(await cursor.fetchone())
+
+    # Mock Qwen API
+    mock_response = MagicMock(spec=httpx.Response)
+    mock_response.status_code = 200
+    mock_response.json.return_value = {
+        "choices": [{"message": {"content": "I've saved the skill. Done."}}]
+    }
+
+    with patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response):
+        await _process_item(item)
+
+    # Verify
+    cursor = await test_db.execute(
+        "SELECT status, result FROM agent_queue WHERE id = ?", (item["id"],)
+    )
+    row = await cursor.fetchone()
+    assert row["status"] == "done"
+    assert "I've saved the skill" in row["result"]
+
+
+@pytest.mark.asyncio
+async def test_agent_worker_handles_missing_video(test_db):
+    """When video doesn't exist, marks as failed gracefully."""
+    from agent_worker import _process_item
+
+    await test_db.execute("""
+        INSERT INTO agent_queue (video_id, user_prompt, status)
+        VALUES ('missing_vid', 'Do something', 'pending')
+    """)
+    await test_db.commit()
+
+    cursor = await test_db.execute(
+        "SELECT id, video_id, user_prompt FROM agent_queue WHERE status = 'pending'"
+    )
+    item = dict(await cursor.fetchone())
+
+    await _process_item(item)
+
+    cursor = await test_db.execute(
+        "SELECT status, result FROM agent_queue WHERE id = ?", (item["id"],)
+    )
+    row = await cursor.fetchone()
+    assert row["status"] == "failed"
+    assert "not found" in row["result"].lower()
+
+
+@pytest.mark.asyncio
+async def test_agent_worker_qwen_error_marks_failed(test_db):
+    """Qwen API failure should mark as failed, not crash."""
+    from agent_worker import _process_item
+
+    await test_db.execute("""
+        INSERT INTO videos (video_id, title, channel_name, url, status)
+        VALUES ('err_vid', 'Error Video', 'Test', 'https://youtube.com/watch?v=err_vid', 'pending')
+    """)
+    await test_db.execute("""
+        INSERT INTO agent_queue (video_id, user_prompt, status)
+        VALUES ('err_vid', 'Do it', 'pending')
+    """)
+    await test_db.commit()
+
+    cursor = await test_db.execute(
+        "SELECT id, video_id, user_prompt FROM agent_queue WHERE status = 'pending'"
+    )
+    item = dict(await cursor.fetchone())
+
+    # Mock HTTP error
+    mock_response = MagicMock(spec=httpx.Response)
+    mock_response.status_code = 503
+    mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
+        "503 Service Unavailable", request=MagicMock(), response=mock_response
+    )
+
+    with patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response):
+        await _process_item(item)
+
+    cursor = await test_db.execute(
+        "SELECT status FROM agent_queue WHERE id = ?", (item["id"],)
+    )
+    row = await cursor.fetchone()
+    assert row["status"] == "failed"