Sprint 002: Ingest duration & view count, FTS search, infinite scroll pagination, and transcript accordion

backend/db.py

@@ -38,11 +38,21 @@ async def init_db():
                 video_id TEXT NOT NULL,
                 user_prompt TEXT NOT NULL,
                 status TEXT DEFAULT 'pending',
+                result TEXT,
                 created_at TEXT DEFAULT CURRENT_TIMESTAMP,
                 processed_at TEXT,
                 FOREIGN KEY (video_id) REFERENCES videos(video_id)
             )
         """)
+        # Migration: Add result column if it doesn't exist (since the DB might already exist)
+        try:
+            async with db.execute("PRAGMA table_info(agent_queue)") as cursor:
+                columns = [row[1] for row in await cursor.fetchall()]
+                if "result" not in columns:
+                    await db.execute("ALTER TABLE agent_queue ADD COLUMN result TEXT")
+        except Exception as e:
+            # Table might not exist yet if not created, but CREATE TABLE IF NOT EXISTS handles it
+            pass
         # FTS5 virtual table for full-text search on title + summary
         await db.execute("""
             CREATE VIRTUAL TABLE IF NOT EXISTS videos_fts USING fts5(

backend/fetcher.py

@@ -6,6 +6,7 @@ import logging
 import aiosqlite
 import httpx
 import xml.etree.ElementTree as ET
+import re
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 from auth import get_credentials
@@ -65,6 +66,20 @@ async def _fetch_rss(client, channel_id):
         return []
 
 
+
+def parse_duration(duration_str: str) -> int:
+    """Parse ISO 8601 duration string (e.g., PT1H2M10S) to seconds."""
+    if not duration_str:
+        return 0
+    pattern = re.compile(r'PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?')
+    match = pattern.match(duration_str)
+    if not match:
+        return 0
+    hours = int(match.group(1)) if match.group(1) else 0
+    minutes = int(match.group(2)) if match.group(2) else 0
+    seconds = int(match.group(3)) if match.group(3) else 0
+    return hours * 3600 + minutes * 60 + seconds
+
 async def fetch_subscriptions():
     """Fetch latest videos from subscribed channels using RSS + YouTube Data API v3."""
     try:
@@ -144,16 +159,22 @@ async def fetch_subscriptions():
         for i in range(0, len(new_ids), 50):
             batch = new_ids[i:i+50]
             try:
-                resp = youtube.videos().list(part="snippet", id=",".join(batch)).execute()
+                resp = youtube.videos().list(part="snippet,contentDetails,statistics", id=",".join(batch)).execute()
                 for item in resp.get("items", []):
                     vid = item["id"]
                     snippet = item["snippet"]
                     cid, rss_title = rss_map.get(vid, (snippet.get("channelId", ""), snippet.get("title", "")))
 
+                    content_details = item.get("contentDetails", {})
+                    statistics = item.get("statistics", {})
+                    duration_str = content_details.get("duration", "")
+                    duration_secs = parse_duration(duration_str)
+                    view_count = int(statistics.get("viewCount", 0))
+
                     await db.execute("""
                         INSERT INTO videos
-                        (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date)
+                        (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration, view_count)
-                        VALUES (?, ?, ?, ?, ?, ?, ?)
+                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                     """, (
                         vid,
                         html.unescape(snippet.get("title", rss_title)),
@@ -161,7 +182,9 @@ async def fetch_subscriptions():
                         snippet.get("channelId", cid),
                         f"https://www.youtube.com/watch?v={vid}",
                         snippet.get("thumbnails", {}).get("high", {}).get("url", ""),
-                        (snippet.get("publishTime") or "")[:10]
+                        (snippet.get("publishTime") or "")[:10],
+                        duration_secs,
+                        view_count
                     ))
                     new_count += 1
             except HttpError as e:

backend/main.py

@@ -107,11 +107,12 @@ async def root():
 async def get_videos(
     status: Optional[str] = "pending",
     limit: int = 50,
+    offset: int = 0,
     channel: Optional[str] = None,
     search: Optional[str] = None,
     db: aiosqlite.Connection = Depends(get_db)
 ):
-    """Get videos with optional search filter."""
+    """Get videos with optional search filter and pagination."""
     if search and len(search) >= 2:
         # Use search endpoint logic
         q = search.replace('"', '\\"')
@@ -123,15 +124,15 @@ async def get_videos(
         if channel:
             base_query += " AND v.channel_name = ?"
             params.append(channel)
-        base_query += " ORDER BY rank LIMIT ?"
+        base_query += " ORDER BY rank LIMIT ? OFFSET ?"
-        params.append(limit)
+        params.extend([limit, offset])
     else:
         if channel:
-            query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
+            query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ? OFFSET ?"
-            cursor = await db.execute(query, (status, channel, limit))
+            cursor = await db.execute(query, (status, channel, limit, offset))
         else:
-            query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
+            query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ? OFFSET ?"
-            cursor = await db.execute(query, (status, limit))
+            cursor = await db.execute(query, (status, limit, offset))
         rows = await cursor.fetchall()
         return [dict(row) for row in rows]
     
@@ -147,8 +148,8 @@ async def get_videos(
         if channel:
             like_query += " AND channel_name = ?"
             params.append(channel)
-        like_query += " ORDER BY publish_date DESC LIMIT ?"
+        like_query += " ORDER BY publish_date DESC LIMIT ? OFFSET ?"
-        params.append(limit)
+        params.extend([limit, offset])
         cursor = await db.execute(like_query, params)
     
     rows = await cursor.fetchall()

backend/tests/test_fetcher.py

@@ -43,8 +43,18 @@ def _mock_youtube_build(sub_channels=None, video_data=None):
                 "channelId": "chan001",
                 "thumbnails": {"high": {"url": f"https://img.youtube.com/vi/{vid}/hqdefault.jpg"}},
                 "publishTime": "2026-06-01T00:00:00Z",
+                "contentDetails": {"duration": "PT5M30S"},
+                "statistics": {"viewCount": "1250"},
+            })
+            snippet = {k: v for k, v in data.items() if k not in ("contentDetails", "statistics")}
+            content_details = data.get("contentDetails", {"duration": "PT5M30S"})
+            statistics = data.get("statistics", {"viewCount": "1250"})
+            items.append({
+                "id": vid,
+                "snippet": snippet,
+                "contentDetails": content_details,
+                "statistics": statistics
             })
-            items.append({"id": vid, "snippet": data})
         return MagicMock(execute=MagicMock(return_value={"items": items}))
 
     svc = MagicMock()
@@ -115,8 +125,12 @@ async def test_fetch_new_videos(test_db):
     assert len(rows) == 2
     assert rows[0]["video_id"] == "vid001"
     assert rows[0]["title"] == "First Video"
+    assert rows[0]["duration"] == 330  # PT5M30S = 330s
+    assert rows[0]["view_count"] == 1250
     assert rows[1]["video_id"] == "vid002"
     assert rows[1]["title"] == "Second Video"
+    assert rows[1]["duration"] == 330
+    assert rows[1]["view_count"] == 1250
 
 
 @pytest.mark.asyncio

docs/specs/sprint-001-sync-reliability.md

@@ -1,32 +1,34 @@
-# Sprint 001 — Sync Reliability & Telegram Delivery
+# Sprint 002 — UX Overhaul, Full-Text Search & Metadata Enrichment
 
-**Dates:** Active
+**Goal:** Overhaul the t-youtube reading experience by completing the search workflow, adding transcript visibility, fetching duration/views metadata, and implementing cursor-based/offset pagination (infinite scroll).
 
-## Goals
+---
 
-- Fix daily sync on NAS
+## T2.1 — Full-Text Search Frontend Integration
-- Verify Telegram delivery pipeline
+- **Files:** `frontend/src/routes/YoutubeDashboard.svelte`
-- Stabilize subscription fetching behind GFW
+- **Done means:** Search queries directly populate the main video list in the sidebar (rather than a temporary dropdown), and clearing the search resets the view.
+- **Verify by:** 
+  1. Type search query -> video list dynamically updates with FTS results from backend.
+  2. Click a search result -> detail view loads it.
+  3. Clear query/blur search -> list reverts to standard tab view.
 
-## Tasks
+## T2.2 — Transcript Accordion in Details View
+- **Files:** `frontend/src/routes/YoutubeDashboard.svelte`
+- **Done means:** Add an expandable accordion below the "AI Digest" section to view the raw video transcript. If transcript is long, render in scrollable container.
+- **Verify by:**
+  1. Click video with transcript -> "Show Full Transcript" accordion appears.
+  2. Click to expand -> raw transcript displays correctly.
 
-### T1: Fix daily sync cron
+## T2.3 — Ingest Duration and View Count Metadata
+- **Files:** `backend/fetcher.py`, `backend/tests/test_fetcher.py`
+- **Done means:** Update `fetcher.py` videos.list API call to request `part="snippet,contentDetails,statistics"`. Parse ISO 8601 duration into integer seconds, and store duration + view count in SQLite db. Update unit tests to include mock response fields.
+- **Verify by:**
+  1. Run `./run_tests.sh` to ensure fetcher tests pass.
+  2. Sync subscriptions -> inspect database to confirm `duration` and `view_count` are populated for new videos.
 
-**Description:**
+## T2.4 — Cursor/Offset Pagination (Infinite Scroll)
-`t-youtube-daily-sync` errored last run. Determine if `run_sync.py` is the active pipeline or if VPS-based `fetch_and_sync.sh` replaced it. Fix whichever is active.
+- **Files:** `backend/main.py`, `frontend/src/routes/YoutubeDashboard.svelte`
-
+- **Done means:** Add pagination queries to `/api/videos` backend endpoint (e.g. `offset` and `limit`). In Svelte frontend, implement scroll listener on the sidebar video list to trigger next page load when reaching bottom.
-**ACs:**
+- **Verify by:**
-- [x] Daily sync completes without errors (verified: last run 2026-07-11 03:16 OK)
+  1. Load pending tab -> initially loads 50 videos.
-- [x] New videos appear in dashboard (verified: DB populated, latest video ingested 2026-07-10)
+  2. Scroll to bottom of sidebar -> next 50 videos append automatically.
-
-**Impl Notes:**
-Check NAS cron logs. VPS sync fires every 6h.
-
-### T2: Verify tg-group-filter delivery
-
-**Description:**
-Cron switched from `deliver: local` to `deliver: all`. Confirm Telegram delivery works for filtered results.
-
-**ACs:**
-- [x] Telegram receives filtered video digest (tg-group-filter running every 30m, verified delivering)
-- [x] No duplicate deliveries (single digest per cycle)

frontend/src/routes/YoutubeDashboard.svelte

@@ -24,6 +24,10 @@
   
   // Search state
   let searchQuery = $state('');
+  let page = $state(0);
+  let hasMore = $state(true);
+  let isMoreLoading = $state(false);
+  let showTranscript = $state(false);
   let searchResults = $state([]);
   let isSearching = $state(false);
   let showSearchResults = $state(false);
@@ -52,13 +56,7 @@
     if (activeTab === 'stats' && !statsData) loadStats();
   });
   
-  $effect(() => {
+
-    if (activeTab === 'pending' && channelFilter) {
-      loadVideos();
-    } else if (activeTab === 'pending' && !channelFilter) {
-      loadVideos();
-    }
-  });
   
   // Auto-generated categories (AI-categorized)
   const AUTO_CATEGORIES = {"AI Engineer":"AI & Tech","AI-Fan AI研究室-帆哥":"AI & Tech","AI大模型":"AI & Tech","AI探索与发现":"AI & Tech","Aurora Tech":"AI & Tech","Claude":"AI & Tech","ColdFusion":"AI & Tech","Data Is Beautiful":"AI & Tech","DeepLearningAI":"AI & Tech","Google":"AI & Tech","Google Cloud Tech":"AI & Tech","Lex Fridman":"AI & Tech","NiceKate AI":"AI & Tech","PC Sim":"AI & Tech","Stanford Online":"AI & Tech","The Linux Foundation":"AI & Tech","回到Axton":"AI & Tech","回形针PaperClip":"AI & Tech","林亦LYi":"AI & Tech","硅谷101播客":"AI & Tech","老石谈芯":"AI & Tech","鲲鹏Talk":"AI & Tech","3D Beast":"3D & Design","Arrimus3D Modeling and Design":"3D & Design","Blender":"3D & Design","Blender Guru":"3D & Design","Creators 3D":"3D & Design","Designfusion":"3D & Design","Foundry":"3D & Design","Grant Abbitt (Gabbitt)":"3D & Design","Gravity Sketch":"3D & Design","Hyperganic":"3D & Design","MSLattice":"3D & Design","Manolo Remiddi":"3D & Design","Rhino 3D (Rhinoceros3d official)":"3D & Design","Shaper3d":"3D & Design","nTop":"3D & Design","spherene AG":"3D & Design","魔界造物":"3D & Design","Chuck Severance":"Programming","Coursera":"Programming","freeCodeCamp.org":"Programming","Kevin Stratvert":"Programming","Leila Gharani":"Programming","LinkedIn Learning":"Programming","Metabase":"Programming","n8n":"Programming","The Coding Gopher":"Programming","Brian Casel":"Programming","code秘密花园":"Programming","cs tech":"Programming","fequalsf":"Programming","openscreencast":"Programming","技术爬爬虾  TechShrimp":"Programming","林本兔Limbuntu":"Programming","Best Choice Review":"Hardware & Reviews","Marques Brownlee":"Hardware & Reviews","TESTV":"Hardware & Reviews","先看评测":"Hardware & Reviews","极客湾Geekerwan":"Hardware & Reviews","硬核拆解":"Hardware & Reviews","笔吧评测室":"Hardware & Reviews","差评硬件部":"Hardware & Reviews","充电头网":"Hardware & Reviews","Apple Developer":"Apple Dev","Made by Google":"Apple Dev","Allen的分享":"Apple Dev","Sypnotix":"Apple Dev","8K World":"Photography","Can Fiona":"Photography","小天fotos":"Photography","影像极客Fotogeeker":"Photography","摄影师PHiL":"Photography","虚空光影CosmosFilm":"Photography","路易斯 LouisDrone":"Photography","Bruno Mars":"Music","GraceLeeMusic":"Music","Katie Melua":"Music","The Weeknd":"Music","TheWeekndVEVO":"Music","杰威爾歌詞MV頻道JVR Lyric MV":"Music","薛汀哲":"Music","Salah Trainer":"Sports","Skills N Talents (swimming)":"Sports","TotalSports TV":"Sports","WorldSBK":"Sports","Everyday Cycling":"Sports","爱羽客羽毛球网":"Sports","adidas":"Sports","Sneaks & Feet极客鞋谈":"Automotive","TOP的遥控玩具":"Automotive","悟空的日常":"Automotive"};
@@ -67,7 +65,9 @@
   
   let channels = $derived([...new Set(videos.map(v => v.channel_name))].sort());
   let filteredVideos = $derived(
-    channelFilter
+    searchQuery.trim().length >= 2
+      ? searchResults
+      : channelFilter
         ? channelFilter.startsWith('📁')
           ? videos.filter(v => (categories[v.channel_name] || '') === channelFilter.replace('📁 ', ''))
           : videos.filter(v => v.channel_name === channelFilter)
@@ -81,10 +81,13 @@
   let searchTimeout = null;
   async function handleSearch(query) {
     searchQuery = query;
-    if (!query || query.length < 2) {
+    if (searchTimeout) clearTimeout(searchTimeout);
+    if (!query || query.trim().length < 2) {
       showSearchResults = false;
+      searchResults = [];
       return;
     }
+    searchTimeout = setTimeout(async () => {
       isSearching = true;
       try {
         const res = await fetch(`/api/videos/search?q=${encodeURIComponent(query)}&limit=50`);
@@ -97,6 +100,21 @@
       } finally {
         isSearching = false;
       }
+    }, 250);
+  }
+
+  async function handleSidebarScroll(e) {
+    if (searchQuery.trim().length >= 2) return;
+    if (isLoading || isMoreLoading || !hasMore) return;
+    const container = e.target;
+    const threshold = 100;
+    const atBottom = (container.scrollHeight - container.scrollTop - container.clientHeight) < threshold;
+    if (atBottom) {
+      isMoreLoading = true;
+      page += 1;
+      await loadVideos(true);
+      isMoreLoading = false;
+    }
   }
   
   // Batch operations
@@ -160,26 +178,43 @@
     document.documentElement.className = theme;
   }
   
-  async function loadVideos() {
+  async function loadVideos(isAppend = false) {
+    if (!isAppend) {
+      page = 0;
+      hasMore = true;
       isLoading = true;
+    }
     try {
+      const limit = 50;
+      const offset = page * limit;
+      
       if (activeTab === 'notes') {
-        // Notes tab: show only pending videos that have notes
+        const params = new URLSearchParams({ status: 'pending', limit: limit.toString(), offset: offset.toString() });
-        const res = await fetch(`/api/videos?status=pending&limit=50`);
+        const res = await fetch(`/api/videos?${params}`);
         if (res.ok) {
-          const all = await res.json();
+          const fetched = await res.json();
-          videos = all.filter(v => noteIds.has(v.video_id));
+          const filtered = fetched.filter(v => noteIds.has(v.video_id));
+          if (isAppend) {
+            videos = [...videos, ...filtered];
+          } else {
+            videos = filtered;
             if (videos.length > 0) selectedVideo = videos[0];
             else selectedVideo = null;
           }
-      } else {
+          if (fetched.length < limit) hasMore = false;
-        const params = new URLSearchParams({status: activeTab, limit: '200'});
+        }
+      } else if (activeTab !== 'stats' && activeTab !== '🤖 Agent') {
+        const params = new URLSearchParams({ status: activeTab, limit: limit.toString(), offset: offset.toString() });
         if (channelFilter && !channelFilter.startsWith('📁')) {
           params.set('channel', channelFilter);
         }
         const res = await fetch(`/api/videos?${params}`);
         if (res.ok) {
-          videos = await res.json();
+          const fetched = await res.json();
+          if (isAppend) {
+            videos = [...videos, ...fetched];
+          } else {
+            videos = fetched;
             if (videos.length > 0) {
               const currentId = selectedVideo?.video_id;
               const found = videos.find(v => v.video_id === currentId);
@@ -188,6 +223,8 @@
               selectedVideo = null;
             }
           }
+          if (fetched.length < limit) hasMore = false;
+        }
       }
     } catch (err) {
       console.error("Error loading videos:", err);
@@ -326,7 +363,17 @@
   
   // ---- Lifecycle ----
   
-  $effect(() => { if (activeTab) loadVideos(); });
+  
+  $effect(() => {
+    if (activeTab && activeTab !== 'stats' && activeTab !== '🤖 Agent') {
+      // Clear search when changing tab or filter
+      searchQuery = '';
+      searchResults = [];
+      showSearchResults = false;
+      loadVideos(false);
+    }
+  });
+
   $effect(() => {
     // Scroll to top when switching videos
     if (selectedVideo && summaryPanel) {
@@ -334,8 +381,7 @@
     }
     const prevId = prevSelectedId;
     prevSelectedId = selectedVideo?.video_id || null;
-    // NOTE: auto-mark-read is now handled by scroll-to-bottom detection (see below)
+    showTranscript = false; // Reset accordion on video switch
-    // and explicit user actions (r/Enter key or Read button).
   });
   
   // Scroll-based auto-mark-read: mark video as read when user scrolls to bottom of summary
@@ -555,15 +601,17 @@
         <div class="px-2 py-1.5" style="border-bottom:1px solid var(--border)">
           <div class="relative">
             <input type="text" 
-                   bind:value={searchQuery}
+                   value={searchQuery}
-                   oninput={handleSearch}
+                   oninput={(e) => handleSearch(e.target.value)}
                    placeholder="Search videos..."
-                   class="w-full text-[11px] p-1.5 rounded border"
+                   class="w-full text-[11px] p-1.5 pr-8 rounded border"
-                   style="background:var(--bg-primary);color:var(--text-primary);border-color:var(--border)"
+                   style="background:var(--bg-primary);color:var(--text-primary);border-color:var(--border)">
-                   onfocus={() => { if (searchResults.length) showSearchResults = true; }}
+            {#if searchQuery}
-                   onblur={() => setTimeout(() => showSearchResults = false, 200)}>
+              <button onclick={() => { searchQuery = ''; searchResults = []; showSearchResults = false; }}
+                      class="absolute right-7 top-2 text-[10px] text-gray-500 hover:text-white transition">✕</button>
+            {/if}
             {#if isSearching}
-              <div class="absolute right-2 top-1.5 text-[9px] animate-spin" style="color:var(--accent)">↻</div>
+              <div class="absolute right-2 top-2 text-[9px] animate-spin" style="color:var(--accent)">↻</div>
             {/if}
           </div>
         </div>
@@ -716,7 +764,7 @@
           </div>
         {/if}
         <!-- Video list -->
-        <div class="flex-1 overflow-y-auto divide-y" style="border-color:var(--border)">
+        <div onscroll={handleSidebarScroll} class="flex-1 overflow-y-auto divide-y" style="border-color:var(--border)">
           {#each filteredVideos as video (video.video_id)}
             {@const i = filteredVideos.indexOf(video)}
             <button onclick={() => { selectedVideo = video; }}
@@ -749,6 +797,9 @@
               </div>
             </button>
           {/each}
+                  {#if isMoreLoading}
+            <div class="p-3 text-center text-[10px]" style="color:var(--text-muted)">Loading more...</div>
+          {/if}
 </div>
       {/if}
     </aside>
@@ -878,6 +929,21 @@ e.g. 'Add this gyroid lattice workflow as a Hermes skill'"
               {@html formatMarkdown(selectedVideo.summary)}
             </div>
           </div>
+
+          <!-- Transcript Accordion -->
+          {#if selectedVideo.transcript && selectedVideo.transcript !== "No transcript available"}
+            <div class="rounded-xl border p-5 space-y-3 shadow-lg" style="background:var(--card-bg);border-color:var(--border)">
+              <div class="flex items-center justify-between cursor-pointer select-none" onclick={() => showTranscript = !showTranscript}>
+                <h2 class="text-xs font-black tracking-widest uppercase" style="color:var(--text-muted)">Video Transcript</h2>
+                <span class="text-xs transition-transform duration-200" style="transform: rotate({showTranscript ? '90deg' : '0deg'})">▶</span>
+              </div>
+              {#if showTranscript}
+                <div class="prose max-w-none text-xs leading-relaxed overflow-y-auto max-h-96 whitespace-pre-wrap pt-2 border-t" style="color:var(--text-secondary);border-color:var(--border)">
+                  {selectedVideo.transcript}
+                </div>
+              {/if}
+            </div>
+          {/if}
         </div>
       {:else}
         <div class="flex-1 flex items-center justify-center" style="color:var(--text-muted)">