Sprint 002: Ingest duration & view count, FTS search, infinite scroll pagination, and transcript accordion

2026-07-11 13:16:59 +08:00
parent 623dff149c
commit 022b024603
6 changed files with 207 additions and 91 deletions
@@ -38,11 +38,21 @@ async def init_db():
                video_id TEXT NOT NULL,
                user_prompt TEXT NOT NULL,
                status TEXT DEFAULT 'pending',
+                result TEXT,
                created_at TEXT DEFAULT CURRENT_TIMESTAMP,
                processed_at TEXT,
                FOREIGN KEY (video_id) REFERENCES videos(video_id)
            )
        """)
+        # Migration: Add result column if it doesn't exist (since the DB might already exist)
+        try:
+            async with db.execute("PRAGMA table_info(agent_queue)") as cursor:
+                columns = [row[1] for row in await cursor.fetchall()]
+                if "result" not in columns:
+                    await db.execute("ALTER TABLE agent_queue ADD COLUMN result TEXT")
+        except Exception as e:
+            # Table might not exist yet if not created, but CREATE TABLE IF NOT EXISTS handles it
+            pass
        # FTS5 virtual table for full-text search on title + summary
        await db.execute("""
            CREATE VIRTUAL TABLE IF NOT EXISTS videos_fts USING fts5(
@@ -6,6 +6,7 @@ import logging
 import aiosqlite
 import httpx
 import xml.etree.ElementTree as ET
+import re
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 from auth import get_credentials
@@ -65,6 +66,20 @@ async def _fetch_rss(client, channel_id):
        return []


+
+def parse_duration(duration_str: str) -> int:
+    """Parse ISO 8601 duration string (e.g., PT1H2M10S) to seconds."""
+    if not duration_str:
+        return 0
+    pattern = re.compile(r'PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?')
+    match = pattern.match(duration_str)
+    if not match:
+        return 0
+    hours = int(match.group(1)) if match.group(1) else 0
+    minutes = int(match.group(2)) if match.group(2) else 0
+    seconds = int(match.group(3)) if match.group(3) else 0
+    return hours * 3600 + minutes * 60 + seconds
+
 async def fetch_subscriptions():
    """Fetch latest videos from subscribed channels using RSS + YouTube Data API v3."""
    try:
@@ -144,16 +159,22 @@ async def fetch_subscriptions():
        for i in range(0, len(new_ids), 50):
            batch = new_ids[i:i+50]
            try:
-                resp = youtube.videos().list(part="snippet", id=",".join(batch)).execute()
+                resp = youtube.videos().list(part="snippet,contentDetails,statistics", id=",".join(batch)).execute()
                for item in resp.get("items", []):
                    vid = item["id"]
                    snippet = item["snippet"]
                    cid, rss_title = rss_map.get(vid, (snippet.get("channelId", ""), snippet.get("title", "")))

+                    content_details = item.get("contentDetails", {})
+                    statistics = item.get("statistics", {})
+                    duration_str = content_details.get("duration", "")
+                    duration_secs = parse_duration(duration_str)
+                    view_count = int(statistics.get("viewCount", 0))
+
                    await db.execute("""
                        INSERT INTO videos
-                        (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date)
-                        VALUES (?, ?, ?, ?, ?, ?, ?)
+                        (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration, view_count)
+                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
                    """, (
                        vid,
                        html.unescape(snippet.get("title", rss_title)),
@@ -161,7 +182,9 @@ async def fetch_subscriptions():
                        snippet.get("channelId", cid),
                        f"https://www.youtube.com/watch?v={vid}",
                        snippet.get("thumbnails", {}).get("high", {}).get("url", ""),
-                        (snippet.get("publishTime") or "")[:10]
+                        (snippet.get("publishTime") or "")[:10],
+                        duration_secs,
+                        view_count
                    ))
                    new_count += 1
            except HttpError as e:
@@ -107,11 +107,12 @@ async def root():
 async def get_videos(
    status: Optional[str] = "pending",
    limit: int = 50,
+    offset: int = 0,
    channel: Optional[str] = None,
    search: Optional[str] = None,
    db: aiosqlite.Connection = Depends(get_db)
 ):
-    """Get videos with optional search filter."""
+    """Get videos with optional search filter and pagination."""
    if search and len(search) >= 2:
        # Use search endpoint logic
        q = search.replace('"', '\\"')
@@ -123,15 +124,15 @@ async def get_videos(
        if channel:
            base_query += " AND v.channel_name = ?"
            params.append(channel)
-        base_query += " ORDER BY rank LIMIT ?"
-        params.append(limit)
+        base_query += " ORDER BY rank LIMIT ? OFFSET ?"
+        params.extend([limit, offset])
    else:
        if channel:
-            query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
-            cursor = await db.execute(query, (status, channel, limit))
+            query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ? OFFSET ?"
+            cursor = await db.execute(query, (status, channel, limit, offset))
        else:
-            query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
-            cursor = await db.execute(query, (status, limit))
+            query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ? OFFSET ?"
+            cursor = await db.execute(query, (status, limit, offset))
        rows = await cursor.fetchall()
        return [dict(row) for row in rows]
    
@@ -147,8 +148,8 @@ async def get_videos(
        if channel:
            like_query += " AND channel_name = ?"
            params.append(channel)
-        like_query += " ORDER BY publish_date DESC LIMIT ?"
-        params.append(limit)
+        like_query += " ORDER BY publish_date DESC LIMIT ? OFFSET ?"
+        params.extend([limit, offset])
        cursor = await db.execute(like_query, params)
    
    rows = await cursor.fetchall()
@@ -43,8 +43,18 @@ def _mock_youtube_build(sub_channels=None, video_data=None):
                "channelId": "chan001",
                "thumbnails": {"high": {"url": f"https://img.youtube.com/vi/{vid}/hqdefault.jpg"}},
                "publishTime": "2026-06-01T00:00:00Z",
+                "contentDetails": {"duration": "PT5M30S"},
+                "statistics": {"viewCount": "1250"},
+            })
+            snippet = {k: v for k, v in data.items() if k not in ("contentDetails", "statistics")}
+            content_details = data.get("contentDetails", {"duration": "PT5M30S"})
+            statistics = data.get("statistics", {"viewCount": "1250"})
+            items.append({
+                "id": vid,
+                "snippet": snippet,
+                "contentDetails": content_details,
+                "statistics": statistics
            })
-            items.append({"id": vid, "snippet": data})
        return MagicMock(execute=MagicMock(return_value={"items": items}))

    svc = MagicMock()
@@ -115,8 +125,12 @@ async def test_fetch_new_videos(test_db):
    assert len(rows) == 2
    assert rows[0]["video_id"] == "vid001"
    assert rows[0]["title"] == "First Video"
+    assert rows[0]["duration"] == 330  # PT5M30S = 330s
+    assert rows[0]["view_count"] == 1250
    assert rows[1]["video_id"] == "vid002"
    assert rows[1]["title"] == "Second Video"
+    assert rows[1]["duration"] == 330
+    assert rows[1]["view_count"] == 1250


@pytest.mark.asyncio
@@ -1,32 +1,34 @@
-# Sprint 001 — Sync Reliability & Telegram Delivery
+# Sprint 002 — UX Overhaul, Full-Text Search & Metadata Enrichment

-**Dates:** Active
+**Goal:** Overhaul the t-youtube reading experience by completing the search workflow, adding transcript visibility, fetching duration/views metadata, and implementing cursor-based/offset pagination (infinite scroll).

-## Goals
+---

- Fix daily sync on NAS
- Verify Telegram delivery pipeline
- Stabilize subscription fetching behind GFW
+## T2.1 — Full-Text Search Frontend Integration
+- **Files:** `frontend/src/routes/YoutubeDashboard.svelte`
+- **Done means:** Search queries directly populate the main video list in the sidebar (rather than a temporary dropdown), and clearing the search resets the view.
+- **Verify by:** 
+  1. Type search query -> video list dynamically updates with FTS results from backend.
+  2. Click a search result -> detail view loads it.
+  3. Clear query/blur search -> list reverts to standard tab view.

-## Tasks
+## T2.2 — Transcript Accordion in Details View
+- **Files:** `frontend/src/routes/YoutubeDashboard.svelte`
+- **Done means:** Add an expandable accordion below the "AI Digest" section to view the raw video transcript. If transcript is long, render in scrollable container.
+- **Verify by:**
+  1. Click video with transcript -> "Show Full Transcript" accordion appears.
+  2. Click to expand -> raw transcript displays correctly.

-### T1: Fix daily sync cron
+## T2.3 — Ingest Duration and View Count Metadata
+- **Files:** `backend/fetcher.py`, `backend/tests/test_fetcher.py`
+- **Done means:** Update `fetcher.py` videos.list API call to request `part="snippet,contentDetails,statistics"`. Parse ISO 8601 duration into integer seconds, and store duration + view count in SQLite db. Update unit tests to include mock response fields.
+- **Verify by:**
+  1. Run `./run_tests.sh` to ensure fetcher tests pass.
+  2. Sync subscriptions -> inspect database to confirm `duration` and `view_count` are populated for new videos.

-**Description:**
-`t-youtube-daily-sync` errored last run. Determine if `run_sync.py` is the active pipeline or if VPS-based `fetch_and_sync.sh` replaced it. Fix whichever is active.
-
-**ACs:**
- [x] Daily sync completes without errors (verified: last run 2026-07-11 03:16 OK)
- [x] New videos appear in dashboard (verified: DB populated, latest video ingested 2026-07-10)
-
-**Impl Notes:**
-Check NAS cron logs. VPS sync fires every 6h.
-
-### T2: Verify tg-group-filter delivery
-
-**Description:**
-Cron switched from `deliver: local` to `deliver: all`. Confirm Telegram delivery works for filtered results.
-
-**ACs:**
- [x] Telegram receives filtered video digest (tg-group-filter running every 30m, verified delivering)
- [x] No duplicate deliveries (single digest per cycle)
+## T2.4 — Cursor/Offset Pagination (Infinite Scroll)
+- **Files:** `backend/main.py`, `frontend/src/routes/YoutubeDashboard.svelte`
+- **Done means:** Add pagination queries to `/api/videos` backend endpoint (e.g. `offset` and `limit`). In Svelte frontend, implement scroll listener on the sidebar video list to trigger next page load when reaching bottom.
+- **Verify by:**
+  1. Load pending tab -> initially loads 50 videos.
+  2. Scroll to bottom of sidebar -> next 50 videos append automatically.
@@ -24,6 +24,10 @@
  
  // Search state
  let searchQuery = $state('');
+  let page = $state(0);
+  let hasMore = $state(true);
+  let isMoreLoading = $state(false);
+  let showTranscript = $state(false);
  let searchResults = $state([]);
  let isSearching = $state(false);
  let showSearchResults = $state(false);
@@ -52,13 +56,7 @@
    if (activeTab === 'stats' && !statsData) loadStats();
  });
  
-  $effect(() => {
-    if (activeTab === 'pending' && channelFilter) {
-      loadVideos();
-    } else if (activeTab === 'pending' && !channelFilter) {
-      loadVideos();
-    }
-  });
+
  
  // Auto-generated categories (AI-categorized)
  const AUTO_CATEGORIES = {"AI Engineer":"AI & Tech","AI-Fan AI研究室-帆哥":"AI & Tech","AI大模型":"AI & Tech","AI探索与发现":"AI & Tech","Aurora Tech":"AI & Tech","Claude":"AI & Tech","ColdFusion":"AI & Tech","Data Is Beautiful":"AI & Tech","DeepLearningAI":"AI & Tech","Google":"AI & Tech","Google Cloud Tech":"AI & Tech","Lex Fridman":"AI & Tech","NiceKate AI":"AI & Tech","PC Sim":"AI & Tech","Stanford Online":"AI & Tech","The Linux Foundation":"AI & Tech","回到Axton":"AI & Tech","回形针PaperClip":"AI & Tech","林亦LYi":"AI & Tech","硅谷101播客":"AI & Tech","老石谈芯":"AI & Tech","鲲鹏Talk":"AI & Tech","3D Beast":"3D & Design","Arrimus3D Modeling and Design":"3D & Design","Blender":"3D & Design","Blender Guru":"3D & Design","Creators 3D":"3D & Design","Designfusion":"3D & Design","Foundry":"3D & Design","Grant Abbitt (Gabbitt)":"3D & Design","Gravity Sketch":"3D & Design","Hyperganic":"3D & Design","MSLattice":"3D & Design","Manolo Remiddi":"3D & Design","Rhino 3D (Rhinoceros3d official)":"3D & Design","Shaper3d":"3D & Design","nTop":"3D & Design","spherene AG":"3D & Design","魔界造物":"3D & Design","Chuck Severance":"Programming","Coursera":"Programming","freeCodeCamp.org":"Programming","Kevin Stratvert":"Programming","Leila Gharani":"Programming","LinkedIn Learning":"Programming","Metabase":"Programming","n8n":"Programming","The Coding Gopher":"Programming","Brian Casel":"Programming","code秘密花园":"Programming","cs tech":"Programming","fequalsf":"Programming","openscreencast":"Programming","技术爬爬虾  TechShrimp":"Programming","林本兔Limbuntu":"Programming","Best Choice Review":"Hardware & Reviews","Marques Brownlee":"Hardware & Reviews","TESTV":"Hardware & Reviews","先看评测":"Hardware & Reviews","极客湾Geekerwan":"Hardware & Reviews","硬核拆解":"Hardware & Reviews","笔吧评测室":"Hardware & Reviews","差评硬件部":"Hardware & Reviews","充电头网":"Hardware & Reviews","Apple Developer":"Apple Dev","Made by Google":"Apple Dev","Allen的分享":"Apple Dev","Sypnotix":"Apple Dev","8K World":"Photography","Can Fiona":"Photography","小天fotos":"Photography","影像极客Fotogeeker":"Photography","摄影师PHiL":"Photography","虚空光影CosmosFilm":"Photography","路易斯 LouisDrone":"Photography","Bruno Mars":"Music","GraceLeeMusic":"Music","Katie Melua":"Music","The Weeknd":"Music","TheWeekndVEVO":"Music","杰威爾歌詞MV頻道JVR Lyric MV":"Music","薛汀哲":"Music","Salah Trainer":"Sports","Skills N Talents (swimming)":"Sports","TotalSports TV":"Sports","WorldSBK":"Sports","Everyday Cycling":"Sports","爱羽客羽毛球网":"Sports","adidas":"Sports","Sneaks & Feet极客鞋谈":"Automotive","TOP的遥控玩具":"Automotive","悟空的日常":"Automotive"};
@@ -67,11 +65,13 @@
  
  let channels = $derived([...new Set(videos.map(v => v.channel_name))].sort());
  let filteredVideos = $derived(
-    channelFilter
-      ? channelFilter.startsWith('📁')
-        ? videos.filter(v => (categories[v.channel_name] || '') === channelFilter.replace('📁 ', ''))
-        : videos.filter(v => v.channel_name === channelFilter)
-      : videos
+    searchQuery.trim().length >= 2
+      ? searchResults
+      : channelFilter
+        ? channelFilter.startsWith('📁')
+          ? videos.filter(v => (categories[v.channel_name] || '') === channelFilter.replace('📁 ', ''))
+          : videos.filter(v => v.channel_name === channelFilter)
+        : videos
  );
  let catList = $derived([...new Set(Object.values(categories).flat())].sort());
  
@@ -81,21 +81,39 @@
  let searchTimeout = null;
  async function handleSearch(query) {
    searchQuery = query;
-    if (!query || query.length < 2) {
+    if (searchTimeout) clearTimeout(searchTimeout);
+    if (!query || query.trim().length < 2) {
      showSearchResults = false;
+      searchResults = [];
      return;
    }
-    isSearching = true;
-    try {
-      const res = await fetch(`/api/videos/search?q=${encodeURIComponent(query)}&limit=50`);
-      if (res.ok) {
-        searchResults = await res.json();
-        showSearchResults = true;
+    searchTimeout = setTimeout(async () => {
+      isSearching = true;
+      try {
+        const res = await fetch(`/api/videos/search?q=${encodeURIComponent(query)}&limit=50`);
+        if (res.ok) {
+          searchResults = await res.json();
+          showSearchResults = true;
+        }
+      } catch(e) {
+        console.error('Search failed:', e);
+      } finally {
+        isSearching = false;
      }
-    } catch(e) {
-      console.error('Search failed:', e);
-    } finally {
-      isSearching = false;
+    }, 250);
+  }
+
+  async function handleSidebarScroll(e) {
+    if (searchQuery.trim().length >= 2) return;
+    if (isLoading || isMoreLoading || !hasMore) return;
+    const container = e.target;
+    const threshold = 100;
+    const atBottom = (container.scrollHeight - container.scrollTop - container.clientHeight) < threshold;
+    if (atBottom) {
+      isMoreLoading = true;
+      page += 1;
+      await loadVideos(true);
+      isMoreLoading = false;
    }
  }
  
@@ -160,33 +178,52 @@
    document.documentElement.className = theme;
  }
  
-  async function loadVideos() {
-    isLoading = true;
+  async function loadVideos(isAppend = false) {
+    if (!isAppend) {
+      page = 0;
+      hasMore = true;
+      isLoading = true;
+    }
    try {
+      const limit = 50;
+      const offset = page * limit;
+      
      if (activeTab === 'notes') {
-        // Notes tab: show only pending videos that have notes
-        const res = await fetch(`/api/videos?status=pending&limit=50`);
+        const params = new URLSearchParams({ status: 'pending', limit: limit.toString(), offset: offset.toString() });
+        const res = await fetch(`/api/videos?${params}`);
        if (res.ok) {
-          const all = await res.json();
-          videos = all.filter(v => noteIds.has(v.video_id));
-          if (videos.length > 0) selectedVideo = videos[0];
-          else selectedVideo = null;
+          const fetched = await res.json();
+          const filtered = fetched.filter(v => noteIds.has(v.video_id));
+          if (isAppend) {
+            videos = [...videos, ...filtered];
+          } else {
+            videos = filtered;
+            if (videos.length > 0) selectedVideo = videos[0];
+            else selectedVideo = null;
+          }
+          if (fetched.length < limit) hasMore = false;
        }
-      } else {
-        const params = new URLSearchParams({status: activeTab, limit: '200'});
+      } else if (activeTab !== 'stats' && activeTab !== '🤖 Agent') {
+        const params = new URLSearchParams({ status: activeTab, limit: limit.toString(), offset: offset.toString() });
        if (channelFilter && !channelFilter.startsWith('📁')) {
          params.set('channel', channelFilter);
        }
        const res = await fetch(`/api/videos?${params}`);
        if (res.ok) {
-          videos = await res.json();
-          if (videos.length > 0) {
-            const currentId = selectedVideo?.video_id;
-            const found = videos.find(v => v.video_id === currentId);
-            selectedVideo = found || videos[0];
+          const fetched = await res.json();
+          if (isAppend) {
+            videos = [...videos, ...fetched];
          } else {
-            selectedVideo = null;
+            videos = fetched;
+            if (videos.length > 0) {
+              const currentId = selectedVideo?.video_id;
+              const found = videos.find(v => v.video_id === currentId);
+              selectedVideo = found || videos[0];
+            } else {
+              selectedVideo = null;
+            }
          }
+          if (fetched.length < limit) hasMore = false;
        }
      }
    } catch (err) {
@@ -326,7 +363,17 @@
  
  // ---- Lifecycle ----
  
-  $effect(() => { if (activeTab) loadVideos(); });
+  
+  $effect(() => {
+    if (activeTab && activeTab !== 'stats' && activeTab !== '🤖 Agent') {
+      // Clear search when changing tab or filter
+      searchQuery = '';
+      searchResults = [];
+      showSearchResults = false;
+      loadVideos(false);
+    }
+  });
+
  $effect(() => {
    // Scroll to top when switching videos
    if (selectedVideo && summaryPanel) {
@@ -334,8 +381,7 @@
    }
    const prevId = prevSelectedId;
    prevSelectedId = selectedVideo?.video_id || null;
-    // NOTE: auto-mark-read is now handled by scroll-to-bottom detection (see below)
-    // and explicit user actions (r/Enter key or Read button).
+    showTranscript = false; // Reset accordion on video switch
  });
  
  // Scroll-based auto-mark-read: mark video as read when user scrolls to bottom of summary
@@ -555,15 +601,17 @@
        <div class="px-2 py-1.5" style="border-bottom:1px solid var(--border)">
          <div class="relative">
            <input type="text" 
-                   bind:value={searchQuery}
-                   oninput={handleSearch}
+                   value={searchQuery}
+                   oninput={(e) => handleSearch(e.target.value)}
                   placeholder="Search videos..."
-                   class="w-full text-[11px] p-1.5 rounded border"
-                   style="background:var(--bg-primary);color:var(--text-primary);border-color:var(--border)"
-                   onfocus={() => { if (searchResults.length) showSearchResults = true; }}
-                   onblur={() => setTimeout(() => showSearchResults = false, 200)}>
+                   class="w-full text-[11px] p-1.5 pr-8 rounded border"
+                   style="background:var(--bg-primary);color:var(--text-primary);border-color:var(--border)">
+            {#if searchQuery}
+              <button onclick={() => { searchQuery = ''; searchResults = []; showSearchResults = false; }}
+                      class="absolute right-7 top-2 text-[10px] text-gray-500 hover:text-white transition">✕</button>
+            {/if}
            {#if isSearching}
-              <div class="absolute right-2 top-1.5 text-[9px] animate-spin" style="color:var(--accent)">↻</div>
+              <div class="absolute right-2 top-2 text-[9px] animate-spin" style="color:var(--accent)">↻</div>
            {/if}
          </div>
        </div>
@@ -716,7 +764,7 @@
          </div>
        {/if}
        <!-- Video list -->
-        <div class="flex-1 overflow-y-auto divide-y" style="border-color:var(--border)">
+        <div onscroll={handleSidebarScroll} class="flex-1 overflow-y-auto divide-y" style="border-color:var(--border)">
          {#each filteredVideos as video (video.video_id)}
            {@const i = filteredVideos.indexOf(video)}
            <button onclick={() => { selectedVideo = video; }}
@@ -749,7 +797,10 @@
              </div>
            </button>
          {/each}
-        </div>
+                  {#if isMoreLoading}
+            <div class="p-3 text-center text-[10px]" style="color:var(--text-muted)">Loading more...</div>
+          {/if}
+</div>
      {/if}
    </aside>

@@ -878,6 +929,21 @@ e.g. 'Add this gyroid lattice workflow as a Hermes skill'"
              {@html formatMarkdown(selectedVideo.summary)}
            </div>
          </div>
+
+          <!-- Transcript Accordion -->
+          {#if selectedVideo.transcript && selectedVideo.transcript !== "No transcript available"}
+            <div class="rounded-xl border p-5 space-y-3 shadow-lg" style="background:var(--card-bg);border-color:var(--border)">
+              <div class="flex items-center justify-between cursor-pointer select-none" onclick={() => showTranscript = !showTranscript}>
+                <h2 class="text-xs font-black tracking-widest uppercase" style="color:var(--text-muted)">Video Transcript</h2>
+                <span class="text-xs transition-transform duration-200" style="transform: rotate({showTranscript ? '90deg' : '0deg'})">▶</span>
+              </div>
+              {#if showTranscript}
+                <div class="prose max-w-none text-xs leading-relaxed overflow-y-auto max-h-96 whitespace-pre-wrap pt-2 border-t" style="color:var(--text-secondary);border-color:var(--border)">
+                  {selectedVideo.transcript}
+                </div>
+              {/if}
+            </div>
+          {/if}
        </div>
      {:else}
        <div class="flex-1 flex items-center justify-center" style="color:var(--text-muted)">