Sprint 002: Ingest duration & view count, FTS search, infinite scroll pagination, and transcript accordion

backend/db.py

@@ -38,11 +38,21 @@ async def init_db():
                 video_id TEXT NOT NULL,
                 user_prompt TEXT NOT NULL,
                 status TEXT DEFAULT 'pending',
+                result TEXT,
                 created_at TEXT DEFAULT CURRENT_TIMESTAMP,
                 processed_at TEXT,
                 FOREIGN KEY (video_id) REFERENCES videos(video_id)
             )
         """)
+        # Migration: Add result column if it doesn't exist (since the DB might already exist)
+        try:
+            async with db.execute("PRAGMA table_info(agent_queue)") as cursor:
+                columns = [row[1] for row in await cursor.fetchall()]
+                if "result" not in columns:
+                    await db.execute("ALTER TABLE agent_queue ADD COLUMN result TEXT")
+        except Exception as e:
+            # Table might not exist yet if not created, but CREATE TABLE IF NOT EXISTS handles it
+            pass
         # FTS5 virtual table for full-text search on title + summary
         await db.execute("""
             CREATE VIRTUAL TABLE IF NOT EXISTS videos_fts USING fts5(

backend/fetcher.py

@@ -6,6 +6,7 @@ import logging
 import aiosqlite
 import httpx
 import xml.etree.ElementTree as ET
+import re
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 from auth import get_credentials
@@ -65,6 +66,20 @@ async def _fetch_rss(client, channel_id):
         return []
 
 
+
+def parse_duration(duration_str: str) -> int:
+    """Parse ISO 8601 duration string (e.g., PT1H2M10S) to seconds."""
+    if not duration_str:
+        return 0
+    pattern = re.compile(r'PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?')
+    match = pattern.match(duration_str)
+    if not match:
+        return 0
+    hours = int(match.group(1)) if match.group(1) else 0
+    minutes = int(match.group(2)) if match.group(2) else 0
+    seconds = int(match.group(3)) if match.group(3) else 0
+    return hours * 3600 + minutes * 60 + seconds
+
 async def fetch_subscriptions():
     """Fetch latest videos from subscribed channels using RSS + YouTube Data API v3."""
     try:
@@ -144,16 +159,22 @@ async def fetch_subscriptions():
         for i in range(0, len(new_ids), 50):
             batch = new_ids[i:i+50]
             try:
-                resp = youtube.videos().list(part="snippet", id=",".join(batch)).execute()
+                resp = youtube.videos().list(part="snippet,contentDetails,statistics", id=",".join(batch)).execute()
                 for item in resp.get("items", []):
                     vid = item["id"]
                     snippet = item["snippet"]
                     cid, rss_title = rss_map.get(vid, (snippet.get("channelId", ""), snippet.get("title", "")))
 
+                    content_details = item.get("contentDetails", {})
+                    statistics = item.get("statistics", {})
+                    duration_str = content_details.get("duration", "")
+                    duration_secs = parse_duration(duration_str)
+                    view_count = int(statistics.get("viewCount", 0))
+
                     await db.execute("""
                         INSERT INTO videos
-                        (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date)
-                        VALUES (?, ?, ?, ?, ?, ?, ?)
+                        (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration, view_count)
+                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                     """, (
                         vid,
                         html.unescape(snippet.get("title", rss_title)),
@@ -161,7 +182,9 @@ async def fetch_subscriptions():
                         snippet.get("channelId", cid),
                         f"https://www.youtube.com/watch?v={vid}",
                         snippet.get("thumbnails", {}).get("high", {}).get("url", ""),
-                        (snippet.get("publishTime") or "")[:10]
+                        (snippet.get("publishTime") or "")[:10],
+                        duration_secs,
+                        view_count
                     ))
                     new_count += 1
             except HttpError as e:

backend/main.py

@@ -107,11 +107,12 @@ async def root():
 async def get_videos(
     status: Optional[str] = "pending",
     limit: int = 50,
+    offset: int = 0,
     channel: Optional[str] = None,
     search: Optional[str] = None,
     db: aiosqlite.Connection = Depends(get_db)
 ):
-    """Get videos with optional search filter."""
+    """Get videos with optional search filter and pagination."""
     if search and len(search) >= 2:
         # Use search endpoint logic
         q = search.replace('"', '\\"')
@@ -123,15 +124,15 @@ async def get_videos(
         if channel:
             base_query += " AND v.channel_name = ?"
             params.append(channel)
-        base_query += " ORDER BY rank LIMIT ?"
-        params.append(limit)
+        base_query += " ORDER BY rank LIMIT ? OFFSET ?"
+        params.extend([limit, offset])
     else:
         if channel:
-            query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
-            cursor = await db.execute(query, (status, channel, limit))
+            query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ? OFFSET ?"
+            cursor = await db.execute(query, (status, channel, limit, offset))
         else:
-            query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
-            cursor = await db.execute(query, (status, limit))
+            query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ? OFFSET ?"
+            cursor = await db.execute(query, (status, limit, offset))
         rows = await cursor.fetchall()
         return [dict(row) for row in rows]
     
@@ -147,8 +148,8 @@ async def get_videos(
         if channel:
             like_query += " AND channel_name = ?"
             params.append(channel)
-        like_query += " ORDER BY publish_date DESC LIMIT ?"
-        params.append(limit)
+        like_query += " ORDER BY publish_date DESC LIMIT ? OFFSET ?"
+        params.extend([limit, offset])
         cursor = await db.execute(like_query, params)
     
     rows = await cursor.fetchall()

backend/tests/test_fetcher.py

@@ -43,8 +43,18 @@ def _mock_youtube_build(sub_channels=None, video_data=None):
                 "channelId": "chan001",
                 "thumbnails": {"high": {"url": f"https://img.youtube.com/vi/{vid}/hqdefault.jpg"}},
                 "publishTime": "2026-06-01T00:00:00Z",
+                "contentDetails": {"duration": "PT5M30S"},
+                "statistics": {"viewCount": "1250"},
+            })
+            snippet = {k: v for k, v in data.items() if k not in ("contentDetails", "statistics")}
+            content_details = data.get("contentDetails", {"duration": "PT5M30S"})
+            statistics = data.get("statistics", {"viewCount": "1250"})
+            items.append({
+                "id": vid,
+                "snippet": snippet,
+                "contentDetails": content_details,
+                "statistics": statistics
             })
-            items.append({"id": vid, "snippet": data})
         return MagicMock(execute=MagicMock(return_value={"items": items}))
 
     svc = MagicMock()
@@ -115,8 +125,12 @@ async def test_fetch_new_videos(test_db):
     assert len(rows) == 2
     assert rows[0]["video_id"] == "vid001"
     assert rows[0]["title"] == "First Video"
+    assert rows[0]["duration"] == 330  # PT5M30S = 330s
+    assert rows[0]["view_count"] == 1250
     assert rows[1]["video_id"] == "vid002"
     assert rows[1]["title"] == "Second Video"
+    assert rows[1]["duration"] == 330
+    assert rows[1]["view_count"] == 1250
 
 
 @pytest.mark.asyncio