diff --git a/backend/db.py b/backend/db.py index d84da04..f96d862 100644 --- a/backend/db.py +++ b/backend/db.py @@ -38,11 +38,21 @@ async def init_db(): video_id TEXT NOT NULL, user_prompt TEXT NOT NULL, status TEXT DEFAULT 'pending', + result TEXT, created_at TEXT DEFAULT CURRENT_TIMESTAMP, processed_at TEXT, FOREIGN KEY (video_id) REFERENCES videos(video_id) ) """) + # Migration: Add result column if it doesn't exist (since the DB might already exist) + try: + async with db.execute("PRAGMA table_info(agent_queue)") as cursor: + columns = [row[1] for row in await cursor.fetchall()] + if "result" not in columns: + await db.execute("ALTER TABLE agent_queue ADD COLUMN result TEXT") + except Exception as e: + # Table might not exist yet if not created, but CREATE TABLE IF NOT EXISTS handles it + pass # FTS5 virtual table for full-text search on title + summary await db.execute(""" CREATE VIRTUAL TABLE IF NOT EXISTS videos_fts USING fts5( diff --git a/backend/fetcher.py b/backend/fetcher.py index c606e26..764a244 100644 --- a/backend/fetcher.py +++ b/backend/fetcher.py @@ -6,6 +6,7 @@ import logging import aiosqlite import httpx import xml.etree.ElementTree as ET +import re from googleapiclient.discovery import build from googleapiclient.errors import HttpError from auth import get_credentials @@ -65,6 +66,20 @@ async def _fetch_rss(client, channel_id): return [] + +def parse_duration(duration_str: str) -> int: + """Parse ISO 8601 duration string (e.g., PT1H2M10S) to seconds.""" + if not duration_str: + return 0 + pattern = re.compile(r'PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?') + match = pattern.match(duration_str) + if not match: + return 0 + hours = int(match.group(1)) if match.group(1) else 0 + minutes = int(match.group(2)) if match.group(2) else 0 + seconds = int(match.group(3)) if match.group(3) else 0 + return hours * 3600 + minutes * 60 + seconds + async def fetch_subscriptions(): """Fetch latest videos from subscribed channels using RSS + YouTube Data API v3.""" try: @@ -144,16 +159,22 @@ async def fetch_subscriptions(): for i in range(0, len(new_ids), 50): batch = new_ids[i:i+50] try: - resp = youtube.videos().list(part="snippet", id=",".join(batch)).execute() + resp = youtube.videos().list(part="snippet,contentDetails,statistics", id=",".join(batch)).execute() for item in resp.get("items", []): vid = item["id"] snippet = item["snippet"] cid, rss_title = rss_map.get(vid, (snippet.get("channelId", ""), snippet.get("title", ""))) + content_details = item.get("contentDetails", {}) + statistics = item.get("statistics", {}) + duration_str = content_details.get("duration", "") + duration_secs = parse_duration(duration_str) + view_count = int(statistics.get("viewCount", 0)) + await db.execute(""" INSERT INTO videos - (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date) - VALUES (?, ?, ?, ?, ?, ?, ?) + (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration, view_count) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( vid, html.unescape(snippet.get("title", rss_title)), @@ -161,7 +182,9 @@ async def fetch_subscriptions(): snippet.get("channelId", cid), f"https://www.youtube.com/watch?v={vid}", snippet.get("thumbnails", {}).get("high", {}).get("url", ""), - (snippet.get("publishTime") or "")[:10] + (snippet.get("publishTime") or "")[:10], + duration_secs, + view_count )) new_count += 1 except HttpError as e: diff --git a/backend/main.py b/backend/main.py index e09511e..a3c8119 100644 --- a/backend/main.py +++ b/backend/main.py @@ -107,11 +107,12 @@ async def root(): async def get_videos( status: Optional[str] = "pending", limit: int = 50, + offset: int = 0, channel: Optional[str] = None, search: Optional[str] = None, db: aiosqlite.Connection = Depends(get_db) ): - """Get videos with optional search filter.""" + """Get videos with optional search filter and pagination.""" if search and len(search) >= 2: # Use search endpoint logic q = search.replace('"', '\\"') @@ -123,15 +124,15 @@ async def get_videos( if channel: base_query += " AND v.channel_name = ?" params.append(channel) - base_query += " ORDER BY rank LIMIT ?" - params.append(limit) + base_query += " ORDER BY rank LIMIT ? OFFSET ?" + params.extend([limit, offset]) else: if channel: - query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?" - cursor = await db.execute(query, (status, channel, limit)) + query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ? OFFSET ?" + cursor = await db.execute(query, (status, channel, limit, offset)) else: - query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?" - cursor = await db.execute(query, (status, limit)) + query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ? OFFSET ?" + cursor = await db.execute(query, (status, limit, offset)) rows = await cursor.fetchall() return [dict(row) for row in rows] @@ -147,8 +148,8 @@ async def get_videos( if channel: like_query += " AND channel_name = ?" params.append(channel) - like_query += " ORDER BY publish_date DESC LIMIT ?" - params.append(limit) + like_query += " ORDER BY publish_date DESC LIMIT ? OFFSET ?" + params.extend([limit, offset]) cursor = await db.execute(like_query, params) rows = await cursor.fetchall() diff --git a/backend/tests/test_fetcher.py b/backend/tests/test_fetcher.py index 6f73023..f9260e5 100644 --- a/backend/tests/test_fetcher.py +++ b/backend/tests/test_fetcher.py @@ -43,8 +43,18 @@ def _mock_youtube_build(sub_channels=None, video_data=None): "channelId": "chan001", "thumbnails": {"high": {"url": f"https://img.youtube.com/vi/{vid}/hqdefault.jpg"}}, "publishTime": "2026-06-01T00:00:00Z", + "contentDetails": {"duration": "PT5M30S"}, + "statistics": {"viewCount": "1250"}, + }) + snippet = {k: v for k, v in data.items() if k not in ("contentDetails", "statistics")} + content_details = data.get("contentDetails", {"duration": "PT5M30S"}) + statistics = data.get("statistics", {"viewCount": "1250"}) + items.append({ + "id": vid, + "snippet": snippet, + "contentDetails": content_details, + "statistics": statistics }) - items.append({"id": vid, "snippet": data}) return MagicMock(execute=MagicMock(return_value={"items": items})) svc = MagicMock() @@ -115,8 +125,12 @@ async def test_fetch_new_videos(test_db): assert len(rows) == 2 assert rows[0]["video_id"] == "vid001" assert rows[0]["title"] == "First Video" + assert rows[0]["duration"] == 330 # PT5M30S = 330s + assert rows[0]["view_count"] == 1250 assert rows[1]["video_id"] == "vid002" assert rows[1]["title"] == "Second Video" + assert rows[1]["duration"] == 330 + assert rows[1]["view_count"] == 1250 @pytest.mark.asyncio diff --git a/docs/specs/sprint-001-sync-reliability.md b/docs/specs/sprint-001-sync-reliability.md index 2de6790..5b44693 100644 --- a/docs/specs/sprint-001-sync-reliability.md +++ b/docs/specs/sprint-001-sync-reliability.md @@ -1,32 +1,34 @@ -# Sprint 001 — Sync Reliability & Telegram Delivery +# Sprint 002 — UX Overhaul, Full-Text Search & Metadata Enrichment -**Dates:** Active +**Goal:** Overhaul the t-youtube reading experience by completing the search workflow, adding transcript visibility, fetching duration/views metadata, and implementing cursor-based/offset pagination (infinite scroll). -## Goals +--- -- Fix daily sync on NAS -- Verify Telegram delivery pipeline -- Stabilize subscription fetching behind GFW +## T2.1 — Full-Text Search Frontend Integration +- **Files:** `frontend/src/routes/YoutubeDashboard.svelte` +- **Done means:** Search queries directly populate the main video list in the sidebar (rather than a temporary dropdown), and clearing the search resets the view. +- **Verify by:** + 1. Type search query -> video list dynamically updates with FTS results from backend. + 2. Click a search result -> detail view loads it. + 3. Clear query/blur search -> list reverts to standard tab view. -## Tasks +## T2.2 — Transcript Accordion in Details View +- **Files:** `frontend/src/routes/YoutubeDashboard.svelte` +- **Done means:** Add an expandable accordion below the "AI Digest" section to view the raw video transcript. If transcript is long, render in scrollable container. +- **Verify by:** + 1. Click video with transcript -> "Show Full Transcript" accordion appears. + 2. Click to expand -> raw transcript displays correctly. -### T1: Fix daily sync cron +## T2.3 — Ingest Duration and View Count Metadata +- **Files:** `backend/fetcher.py`, `backend/tests/test_fetcher.py` +- **Done means:** Update `fetcher.py` videos.list API call to request `part="snippet,contentDetails,statistics"`. Parse ISO 8601 duration into integer seconds, and store duration + view count in SQLite db. Update unit tests to include mock response fields. +- **Verify by:** + 1. Run `./run_tests.sh` to ensure fetcher tests pass. + 2. Sync subscriptions -> inspect database to confirm `duration` and `view_count` are populated for new videos. -**Description:** -`t-youtube-daily-sync` errored last run. Determine if `run_sync.py` is the active pipeline or if VPS-based `fetch_and_sync.sh` replaced it. Fix whichever is active. - -**ACs:** -- [x] Daily sync completes without errors (verified: last run 2026-07-11 03:16 OK) -- [x] New videos appear in dashboard (verified: DB populated, latest video ingested 2026-07-10) - -**Impl Notes:** -Check NAS cron logs. VPS sync fires every 6h. - -### T2: Verify tg-group-filter delivery - -**Description:** -Cron switched from `deliver: local` to `deliver: all`. Confirm Telegram delivery works for filtered results. - -**ACs:** -- [x] Telegram receives filtered video digest (tg-group-filter running every 30m, verified delivering) -- [x] No duplicate deliveries (single digest per cycle) +## T2.4 — Cursor/Offset Pagination (Infinite Scroll) +- **Files:** `backend/main.py`, `frontend/src/routes/YoutubeDashboard.svelte` +- **Done means:** Add pagination queries to `/api/videos` backend endpoint (e.g. `offset` and `limit`). In Svelte frontend, implement scroll listener on the sidebar video list to trigger next page load when reaching bottom. +- **Verify by:** + 1. Load pending tab -> initially loads 50 videos. + 2. Scroll to bottom of sidebar -> next 50 videos append automatically. diff --git a/frontend/src/routes/YoutubeDashboard.svelte b/frontend/src/routes/YoutubeDashboard.svelte index dd55979..6627f20 100644 --- a/frontend/src/routes/YoutubeDashboard.svelte +++ b/frontend/src/routes/YoutubeDashboard.svelte @@ -24,6 +24,10 @@ // Search state let searchQuery = $state(''); + let page = $state(0); + let hasMore = $state(true); + let isMoreLoading = $state(false); + let showTranscript = $state(false); let searchResults = $state([]); let isSearching = $state(false); let showSearchResults = $state(false); @@ -52,13 +56,7 @@ if (activeTab === 'stats' && !statsData) loadStats(); }); - $effect(() => { - if (activeTab === 'pending' && channelFilter) { - loadVideos(); - } else if (activeTab === 'pending' && !channelFilter) { - loadVideos(); - } - }); + // Auto-generated categories (AI-categorized) const AUTO_CATEGORIES = {"AI Engineer":"AI & Tech","AI-Fan AI研究室-帆哥":"AI & Tech","AI大模型":"AI & Tech","AI探索与发现":"AI & Tech","Aurora Tech":"AI & Tech","Claude":"AI & Tech","ColdFusion":"AI & Tech","Data Is Beautiful":"AI & Tech","DeepLearningAI":"AI & Tech","Google":"AI & Tech","Google Cloud Tech":"AI & Tech","Lex Fridman":"AI & Tech","NiceKate AI":"AI & Tech","PC Sim":"AI & Tech","Stanford Online":"AI & Tech","The Linux Foundation":"AI & Tech","回到Axton":"AI & Tech","回形针PaperClip":"AI & Tech","林亦LYi":"AI & Tech","硅谷101播客":"AI & Tech","老石谈芯":"AI & Tech","鲲鹏Talk":"AI & Tech","3D Beast":"3D & Design","Arrimus3D Modeling and Design":"3D & Design","Blender":"3D & Design","Blender Guru":"3D & Design","Creators 3D":"3D & Design","Designfusion":"3D & Design","Foundry":"3D & Design","Grant Abbitt (Gabbitt)":"3D & Design","Gravity Sketch":"3D & Design","Hyperganic":"3D & Design","MSLattice":"3D & Design","Manolo Remiddi":"3D & Design","Rhino 3D (Rhinoceros3d official)":"3D & Design","Shaper3d":"3D & Design","nTop":"3D & Design","spherene AG":"3D & Design","魔界造物":"3D & Design","Chuck Severance":"Programming","Coursera":"Programming","freeCodeCamp.org":"Programming","Kevin Stratvert":"Programming","Leila Gharani":"Programming","LinkedIn Learning":"Programming","Metabase":"Programming","n8n":"Programming","The Coding Gopher":"Programming","Brian Casel":"Programming","code秘密花园":"Programming","cs tech":"Programming","fequalsf":"Programming","openscreencast":"Programming","技术爬爬虾 TechShrimp":"Programming","林本兔Limbuntu":"Programming","Best Choice Review":"Hardware & Reviews","Marques Brownlee":"Hardware & Reviews","TESTV":"Hardware & Reviews","先看评测":"Hardware & Reviews","极客湾Geekerwan":"Hardware & Reviews","硬核拆解":"Hardware & Reviews","笔吧评测室":"Hardware & Reviews","差评硬件部":"Hardware & Reviews","充电头网":"Hardware & Reviews","Apple Developer":"Apple Dev","Made by Google":"Apple Dev","Allen的分享":"Apple Dev","Sypnotix":"Apple Dev","8K World":"Photography","Can Fiona":"Photography","小天fotos":"Photography","影像极客Fotogeeker":"Photography","摄影师PHiL":"Photography","虚空光影CosmosFilm":"Photography","路易斯 LouisDrone":"Photography","Bruno Mars":"Music","GraceLeeMusic":"Music","Katie Melua":"Music","The Weeknd":"Music","TheWeekndVEVO":"Music","杰威爾歌詞MV頻道JVR Lyric MV":"Music","薛汀哲":"Music","Salah Trainer":"Sports","Skills N Talents (swimming)":"Sports","TotalSports TV":"Sports","WorldSBK":"Sports","Everyday Cycling":"Sports","爱羽客羽毛球网":"Sports","adidas":"Sports","Sneaks & Feet极客鞋谈":"Automotive","TOP的遥控玩具":"Automotive","悟空的日常":"Automotive"}; @@ -67,11 +65,13 @@ let channels = $derived([...new Set(videos.map(v => v.channel_name))].sort()); let filteredVideos = $derived( - channelFilter - ? channelFilter.startsWith('📁') - ? videos.filter(v => (categories[v.channel_name] || '') === channelFilter.replace('📁 ', '')) - : videos.filter(v => v.channel_name === channelFilter) - : videos + searchQuery.trim().length >= 2 + ? searchResults + : channelFilter + ? channelFilter.startsWith('📁') + ? videos.filter(v => (categories[v.channel_name] || '') === channelFilter.replace('📁 ', '')) + : videos.filter(v => v.channel_name === channelFilter) + : videos ); let catList = $derived([...new Set(Object.values(categories).flat())].sort()); @@ -81,21 +81,39 @@ let searchTimeout = null; async function handleSearch(query) { searchQuery = query; - if (!query || query.length < 2) { + if (searchTimeout) clearTimeout(searchTimeout); + if (!query || query.trim().length < 2) { showSearchResults = false; + searchResults = []; return; } - isSearching = true; - try { - const res = await fetch(`/api/videos/search?q=${encodeURIComponent(query)}&limit=50`); - if (res.ok) { - searchResults = await res.json(); - showSearchResults = true; + searchTimeout = setTimeout(async () => { + isSearching = true; + try { + const res = await fetch(`/api/videos/search?q=${encodeURIComponent(query)}&limit=50`); + if (res.ok) { + searchResults = await res.json(); + showSearchResults = true; + } + } catch(e) { + console.error('Search failed:', e); + } finally { + isSearching = false; } - } catch(e) { - console.error('Search failed:', e); - } finally { - isSearching = false; + }, 250); + } + + async function handleSidebarScroll(e) { + if (searchQuery.trim().length >= 2) return; + if (isLoading || isMoreLoading || !hasMore) return; + const container = e.target; + const threshold = 100; + const atBottom = (container.scrollHeight - container.scrollTop - container.clientHeight) < threshold; + if (atBottom) { + isMoreLoading = true; + page += 1; + await loadVideos(true); + isMoreLoading = false; } } @@ -160,33 +178,52 @@ document.documentElement.className = theme; } - async function loadVideos() { - isLoading = true; + async function loadVideos(isAppend = false) { + if (!isAppend) { + page = 0; + hasMore = true; + isLoading = true; + } try { + const limit = 50; + const offset = page * limit; + if (activeTab === 'notes') { - // Notes tab: show only pending videos that have notes - const res = await fetch(`/api/videos?status=pending&limit=50`); + const params = new URLSearchParams({ status: 'pending', limit: limit.toString(), offset: offset.toString() }); + const res = await fetch(`/api/videos?${params}`); if (res.ok) { - const all = await res.json(); - videos = all.filter(v => noteIds.has(v.video_id)); - if (videos.length > 0) selectedVideo = videos[0]; - else selectedVideo = null; + const fetched = await res.json(); + const filtered = fetched.filter(v => noteIds.has(v.video_id)); + if (isAppend) { + videos = [...videos, ...filtered]; + } else { + videos = filtered; + if (videos.length > 0) selectedVideo = videos[0]; + else selectedVideo = null; + } + if (fetched.length < limit) hasMore = false; } - } else { - const params = new URLSearchParams({status: activeTab, limit: '200'}); + } else if (activeTab !== 'stats' && activeTab !== '🤖 Agent') { + const params = new URLSearchParams({ status: activeTab, limit: limit.toString(), offset: offset.toString() }); if (channelFilter && !channelFilter.startsWith('📁')) { params.set('channel', channelFilter); } const res = await fetch(`/api/videos?${params}`); if (res.ok) { - videos = await res.json(); - if (videos.length > 0) { - const currentId = selectedVideo?.video_id; - const found = videos.find(v => v.video_id === currentId); - selectedVideo = found || videos[0]; + const fetched = await res.json(); + if (isAppend) { + videos = [...videos, ...fetched]; } else { - selectedVideo = null; + videos = fetched; + if (videos.length > 0) { + const currentId = selectedVideo?.video_id; + const found = videos.find(v => v.video_id === currentId); + selectedVideo = found || videos[0]; + } else { + selectedVideo = null; + } } + if (fetched.length < limit) hasMore = false; } } } catch (err) { @@ -326,7 +363,17 @@ // ---- Lifecycle ---- - $effect(() => { if (activeTab) loadVideos(); }); + + $effect(() => { + if (activeTab && activeTab !== 'stats' && activeTab !== '🤖 Agent') { + // Clear search when changing tab or filter + searchQuery = ''; + searchResults = []; + showSearchResults = false; + loadVideos(false); + } + }); + $effect(() => { // Scroll to top when switching videos if (selectedVideo && summaryPanel) { @@ -334,8 +381,7 @@ } const prevId = prevSelectedId; prevSelectedId = selectedVideo?.video_id || null; - // NOTE: auto-mark-read is now handled by scroll-to-bottom detection (see below) - // and explicit user actions (r/Enter key or Read button). + showTranscript = false; // Reset accordion on video switch }); // Scroll-based auto-mark-read: mark video as read when user scrolls to bottom of summary @@ -555,15 +601,17 @@
handleSearch(e.target.value)} placeholder="Search videos..." - class="w-full text-[11px] p-1.5 rounded border" - style="background:var(--bg-primary);color:var(--text-primary);border-color:var(--border)" - onfocus={() => { if (searchResults.length) showSearchResults = true; }} - onblur={() => setTimeout(() => showSearchResults = false, 200)}> + class="w-full text-[11px] p-1.5 pr-8 rounded border" + style="background:var(--bg-primary);color:var(--text-primary);border-color:var(--border)"> + {#if searchQuery} + + {/if} {#if isSearching} -
↻
+
↻
{/if}
@@ -716,7 +764,7 @@ {/if} -
+
{#each filteredVideos as video (video.video_id)} {@const i = filteredVideos.indexOf(video)}
{/each} -
+ {#if isMoreLoading} +
Loading more...
+ {/if} + {/if} @@ -878,6 +929,21 @@ e.g. 'Add this gyroid lattice workflow as a Hermes skill'" {@html formatMarkdown(selectedVideo.summary)} + + + {#if selectedVideo.transcript && selectedVideo.transcript !== "No transcript available"} +
+
showTranscript = !showTranscript}> +

Video Transcript

+ ▶ +
+ {#if showTranscript} +
+ {selectedVideo.transcript} +
+ {/if} +
+ {/if} {:else}