From a154bc7978904c474b058efdb03da8aa415ff72a Mon Sep 17 00:00:00 2001 From: "Gan, Jimmy" Date: Wed, 8 Jul 2026 02:23:00 +0800 Subject: [PATCH] feat: notes + agent queue, frontend rebuild, pip mirror, gitignore sync_rotation.json - Add video_notes table and agent_queue table for persistent notes and AI agent tasks - Add API endpoints: POST/GET /api/notes/{id}, POST /api/notes/migrate, agent queue endpoints - Rebuild frontend: fix duplicate loadVideos(), non-blocking localStorage migration, notes UI - Switch pip install to Tsinghua mirror for faster downloads in China - Add backend/sync_rotation.json to .gitignore (runtime state) --- .gitignore | 1 + Dockerfile | 5 +- backend/db.py | 19 + backend/main.py | 151 +++++++- backend/requirements.txt | 1 - frontend/src/routes/YoutubeDashboard.svelte | 374 ++++++++++++++------ 6 files changed, 434 insertions(+), 117 deletions(-) diff --git a/.gitignore b/.gitignore index 27c4b97..795567b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ config/cookies.txt token.pickle config/client_secret.json config/token.pickle +backend/sync_rotation.json diff --git a/Dockerfile b/Dockerfile index 8f490c0..b19a9da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,13 +16,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ffmpeg \ && rm -rf /var/lib/apt/lists/* -# Download latest official yt-dlp to ensure subscription scrapers do not break -RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp && \ - chmod a+rx /usr/local/bin/yt-dlp # Install python packages COPY backend/requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt # Copy built static frontend COPY --from=frontend-builder /app/frontend/dist ./frontend/dist diff --git a/backend/db.py b/backend/db.py index a92b7df..76098db 100644 --- a/backend/db.py +++ b/backend/db.py @@ -23,6 +23,25 @@ async def init_db(): created_at TEXT DEFAULT CURRENT_TIMESTAMP ) """) + await db.execute(""" + CREATE TABLE IF NOT EXISTS video_notes ( + video_id TEXT PRIMARY KEY, + note_text TEXT NOT NULL, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (video_id) REFERENCES videos(video_id) + ) + """) + await db.execute(""" + CREATE TABLE IF NOT EXISTS agent_queue ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + video_id TEXT NOT NULL, + user_prompt TEXT NOT NULL, + status TEXT DEFAULT 'pending', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + processed_at TEXT, + FOREIGN KEY (video_id) REFERENCES videos(video_id) + ) + """) await db.commit() async def get_db(): diff --git a/backend/main.py b/backend/main.py index 1596570..3e82548 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,4 +1,5 @@ import os +import json import logging from fastapi import FastAPI, BackgroundTasks, Depends, HTTPException, Query from fastapi.middleware.cors import CORSMiddleware @@ -16,7 +17,6 @@ log = logging.getLogger(__name__) app = FastAPI(title="T YouTube", description="Text-based YouTube Summarizer") -# Enable CORS for development app.add_middleware( CORSMiddleware, allow_origins=["*"], @@ -28,7 +28,20 @@ app.add_middleware( class StatusUpdate(BaseModel): status: str -# Database Startup Hook +class NoteUpdate(BaseModel): + note_text: str + queue_to_agent: bool = False + +class NoteMigrate(BaseModel): + notes: dict + +class AgentQueueItem(BaseModel): + video_id: str + user_prompt: str + +class AgentQueueUpdate(BaseModel): + status: str + @app.on_event("startup") async def startup(): await init_db() @@ -39,7 +52,6 @@ async def health(): @app.get("/auto_categories.json") async def auto_categories(): - import json, os path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "frontend", "src", "auto_categories.json") if os.path.exists(path): with open(path) as f: @@ -93,7 +105,6 @@ async def trigger_summarize(video_id: str, background_tasks: BackgroundTasks, db exists = await cursor.fetchone() if not exists: raise HTTPException(status_code=404, detail="Video not found") - background_tasks.add_task(summarize_video, video_id) return {"message": f"Summarization for video {video_id} queued."} @@ -105,17 +116,143 @@ async def update_status( ): if update.status not in ["pending", "read", "skipped"]: raise HTTPException(status_code=400, detail="Invalid status value") - cursor = await db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,)) exists = await cursor.fetchone() if not exists: raise HTTPException(status_code=404, detail="Video not found") - await db.execute("UPDATE videos SET status = ? WHERE video_id = ?", (update.status, video_id)) await db.commit() return {"message": f"Video {video_id} status updated to {update.status}."} -# Mount frontend built files for production if they exist +# ---- Notes (server-side, replaces localStorage) ---- + + + +@app.get("/api/notes") +async def list_notes(db: aiosqlite.Connection = Depends(get_db)): + cursor = await db.execute("SELECT video_id, note_text, updated_at FROM video_notes ORDER BY updated_at DESC") + rows = await cursor.fetchall() + return [dict(row) for row in rows] + +@app.get("/api/notes/{video_id}") +async def get_note(video_id: str, db: aiosqlite.Connection = Depends(get_db)): + cursor = await db.execute("SELECT note_text, updated_at FROM video_notes WHERE video_id = ?", (video_id,)) + row = await cursor.fetchone() + if not row: + return {"note_text": "", "updated_at": None} + return dict(row) + +@app.post("/api/notes/{video_id}") +async def save_note(video_id: str, update: NoteUpdate, db: aiosqlite.Connection = Depends(get_db)): + await db.execute(""" + INSERT INTO video_notes (video_id, note_text, updated_at) + VALUES (?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(video_id) DO UPDATE SET + note_text = excluded.note_text, + updated_at = CURRENT_TIMESTAMP + """, (video_id, update.note_text)) + await db.commit() + + if update.queue_to_agent and update.note_text.strip(): + await db.execute(""" + INSERT INTO agent_queue (video_id, user_prompt, status) + VALUES (?, ?, 'pending') + """, (video_id, update.note_text.strip())) + await db.commit() + log.info(f"Queued video {video_id} for agent: {update.note_text[:80]}") + + return {"message": "Note saved"} + +@app.post("/api/notes/migrate") +async def migrate_notes(migration: NoteMigrate, db: aiosqlite.Connection = Depends(get_db)): + count = 0 + for video_id, note_text in migration.notes.items(): + if not note_text.strip(): + continue + await db.execute(""" + INSERT INTO video_notes (video_id, note_text, updated_at) + VALUES (?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(video_id) DO UPDATE SET + note_text = excluded.note_text, + updated_at = CURRENT_TIMESTAMP + """, (video_id, note_text)) + count += 1 + await db.commit() + return {"message": f"{count} notes migrated"} + +# ---- Agent Queue ---- + +@app.get("/api/agent-queue") +async def list_agent_queue( + status: Optional[str] = None, + limit: int = 50, + db: aiosqlite.Connection = Depends(get_db) +): + if status: + query = """ + SELECT q.id, q.video_id, q.user_prompt, q.status, q.created_at, q.processed_at, + v.title, v.channel_name, v.url, v.summary + FROM agent_queue q + LEFT JOIN videos v ON q.video_id = v.video_id + WHERE q.status = ? + ORDER BY q.created_at DESC LIMIT ? + """ + cursor = await db.execute(query, (status, limit)) + else: + query = """ + SELECT q.id, q.video_id, q.user_prompt, q.status, q.created_at, q.processed_at, + v.title, v.channel_name, v.url, v.summary + FROM agent_queue q + LEFT JOIN videos v ON q.video_id = v.video_id + ORDER BY q.created_at DESC LIMIT ? + """ + cursor = await db.execute(query, (limit,)) + rows = await cursor.fetchall() + return [dict(row) for row in rows] + +@app.post("/api/agent-queue") +async def add_to_agent_queue(item: AgentQueueItem, db: aiosqlite.Connection = Depends(get_db)): + cursor = await db.execute("SELECT 1 FROM videos WHERE video_id = ?", (item.video_id,)) + exists = await cursor.fetchone() + if not exists: + raise HTTPException(status_code=404, detail="Video not found") + await db.execute(""" + INSERT INTO agent_queue (video_id, user_prompt, status) + VALUES (?, ?, 'pending') + """, (item.video_id, item.user_prompt)) + await db.commit() + return {"message": "Item queued for agent"} + +@app.patch("/api/agent-queue/{item_id}") +async def update_agent_queue(item_id: int, update: AgentQueueUpdate, db: aiosqlite.Connection = Depends(get_db)): + valid_statuses = ["pending", "processing", "done", "failed"] + if update.status not in valid_statuses: + raise HTTPException(status_code=400, detail=f"Invalid status. Must be one of: {valid_statuses}") + cursor = await db.execute("SELECT 1 FROM agent_queue WHERE id = ?", (item_id,)) + exists = await cursor.fetchone() + if not exists: + raise HTTPException(status_code=404, detail="Queue item not found") + if update.status in ("done", "failed"): + await db.execute(""" + UPDATE agent_queue SET status = ?, processed_at = CURRENT_TIMESTAMP WHERE id = ? + """, (update.status, item_id)) + else: + await db.execute("UPDATE agent_queue SET status = ? WHERE id = ?", (update.status, item_id)) + await db.commit() + return {"message": f"Queue item {item_id} updated to {update.status}"} + +@app.delete("/api/agent-queue/{item_id}") +async def remove_from_agent_queue(item_id: int, db: aiosqlite.Connection = Depends(get_db)): + cursor = await db.execute("SELECT 1 FROM agent_queue WHERE id = ?", (item_id,)) + exists = await cursor.fetchone() + if not exists: + raise HTTPException(status_code=404, detail="Queue item not found") + await db.execute("DELETE FROM agent_queue WHERE id = ?", (item_id,)) + await db.commit() + return {"message": f"Queue item {item_id} removed"} + +# ---- Static frontend ---- + frontend_build_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "frontend/dist") if os.path.exists(frontend_build_path): app.mount("/", StaticFiles(directory=frontend_build_path, html=True), name="frontend") diff --git a/backend/requirements.txt b/backend/requirements.txt index 3c6f128..b448e0c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,6 +1,5 @@ fastapi>=0.100.0 uvicorn>=0.20.0 -yt-dlp>=2023.0.0 youtube-transcript-api>=0.6.0 aiosqlite>=0.18.0 httpx>=0.24.0 diff --git a/frontend/src/routes/YoutubeDashboard.svelte b/frontend/src/routes/YoutubeDashboard.svelte index 145a8d4..82ec240 100644 --- a/frontend/src/routes/YoutubeDashboard.svelte +++ b/frontend/src/routes/YoutubeDashboard.svelte @@ -8,16 +8,20 @@ let isSyncing = $state(false); let syncMessage = $state(''); let pendingCount = $state(0); - let nextSync = $state(localStorage.getItem('t-yt-next-sync') || ''); let theme = $state(localStorage.getItem('theme') || 'dark'); let summaryPanel = $state(null); - let notes = $state(JSON.parse(localStorage.getItem('t-yt-notes') || '{}')); + let notes = $state({}); let activeNote = $state(null); + let queueToAgent = $state(false); let showHelp = $state(false); let channelFilter = $state(''); let allChannels = $state([]); let editingCategories = $state(false); + // Agent queue state + let agentItems = $state([]); + let agentLoading = $state(false); + $effect(() => { if (activeTab === 'pending' && channelFilter) { loadVideos(); @@ -41,6 +45,8 @@ ); let catList = $derived([...new Set(Object.values(categories).flat())].sort()); + let noteIds = $derived(new Set(Object.keys(notes).filter(k => notes[k] && notes[k].trim()))); + function toggleTheme() { theme = theme === 'dark' ? 'light' : 'dark'; localStorage.setItem('theme', theme); @@ -51,11 +57,11 @@ isLoading = true; try { if (activeTab === 'notes') { - // Notes tab: show all pending videos, but only those with notes + // Notes tab: show only pending videos that have notes const res = await fetch(`/api/videos?status=pending&limit=50`); if (res.ok) { const all = await res.json(); - videos = all.filter(v => notes[v.video_id]); + videos = all.filter(v => noteIds.has(v.video_id)); if (videos.length > 0) selectedVideo = videos[0]; else selectedVideo = null; } @@ -103,7 +109,6 @@ if (!silent) { videos = videos.filter(v => v.video_id !== videoId); if (selectedVideo?.video_id === videoId) selectedVideo = videos[0] || null; - // Reload when list gets short if (videos.length < 20) loadVideos(); } } @@ -114,11 +119,78 @@ setTimeout(() => loadVideos(), 3000); } + // ---- Server-side notes ---- + + async function loadNotes() { + try { + const res = await fetch('/api/notes'); + if (res.ok) { + const all = await res.json(); + const map = {}; + for (const item of all) map[item.video_id] = item.note_text; + notes = map; + } + } catch(e) { console.error('Failed to load notes:', e); } + } + + async function migrateLocalNotes() { + try { + const local = JSON.parse(localStorage.getItem('t-yt-notes') || '{}'); + const keys = Object.keys(local).filter(k => local[k].trim()); + if (keys.length === 0) return; + const missing = keys.filter(k => !notes[k]); + if (missing.length === 0) return; + const payload = {}; + for (const k of missing) payload[k] = local[k]; + await fetch('/api/notes/migrate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ notes: payload }) + }); + await loadNotes(); + } catch(e) {} + } + + async function saveNote() { + const vid = selectedVideo?.video_id; + if (!vid) return; + const textarea = document.querySelector('#note-textarea'); + const current = textarea ? textarea.value : (notes[vid] || ''); + if (!current.trim()) return; + // Save to server + await fetch(`/api/notes/${vid}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ note_text: current, queue_to_agent: queueToAgent }) + }); + // Update local state + notes = {...notes, [vid]: current}; + activeNote = null; + queueToAgent = false; + } + + // ---- Agent queue ---- + + async function loadAgentQueue() { + agentLoading = true; + try { + const res = await fetch('/api/agent-queue'); + if (res.ok) agentItems = await res.json(); + } catch(e) { console.error('Failed to load agent queue:', e); } + finally { agentLoading = false; } + } + + async function removeAgentItem(id) { + await fetch(`/api/agent-queue/${id}`, { method: 'DELETE' }); + agentItems = agentItems.filter(i => i.id !== id); + } + + // ---- Lifecycle ---- + $effect(() => { if (activeTab) loadVideos(); }); $effect(() => { if (selectedVideo && summaryPanel) { tick().then(() => summaryPanel.scrollTop = 0); - // Auto-mark as read after 3s of viewing a summarized video const vid = selectedVideo.video_id; const hasSummary = selectedVideo.summary && !selectedVideo.summary.includes('Error') && !selectedVideo.summary.includes('📺 Short clip'); if (hasSummary && activeTab === 'pending') { @@ -127,9 +199,17 @@ } } }); - onMount(() => { + $effect(() => { + if (activeTab === 'agent') loadAgentQueue(); + }); + + onMount(async () => { document.documentElement.className = theme; - loadVideos(); + await loadNotes(); + // Migrate localStorage notes to server (non-blocking) + migrateLocalNotes(); + await loadVideos(); + loaded = true; loadPendingCount(); updateSyncTimer(); loadChannels(); @@ -142,7 +222,7 @@ if (!el) return; const now = new Date(); const next = new Date(now); - next.setHours(3, 0, 0, 0); // 3 AM daily + next.setHours(3, 0, 0, 0); if (now > next) next.setDate(next.getDate() + 1); const diff = next - now; const h = Math.floor(diff / 3600000); @@ -194,8 +274,7 @@ } else if (e.key === 'Enter') { e.preventDefault(); if (activeNote && selectedVideo && activeNote === selectedVideo.video_id) { - localStorage.setItem('t-yt-notes', JSON.stringify(notes)); - activeNote = null; + saveNote(); } else if (selectedVideo) { updateStatus(selectedVideo.video_id, 'read'); } @@ -220,20 +299,6 @@ requestAnimationFrame(() => node.focus()); } - function saveNote() { - const vid = selectedVideo?.video_id; - if (!vid) return; - const prev = notes[vid] || ''; - const textarea = document.querySelector('#note-textarea'); - const current = textarea ? textarea.value : prev; - if (!current.trim()) return; - const ts = new Date().toLocaleString(); - const stamped = `[${ts}] ${current.trim()}\n${prev}`; - notes = {...notes, [vid]: stamped}; - localStorage.setItem('t-yt-notes', JSON.stringify(notes)); - activeNote = null; - } - function formatMarkdown(text) { if (!text) return '

No summary yet.

'; let html = text.replace(/\*\*(.*?)\*\*/g, '$1'); @@ -245,10 +310,19 @@ }).join(''); return html; } + + function noteWordCount(noteText) { + if (!noteText) return 0; + return noteText.trim().split(/\s+/).length; + } + + function formatDate(d) { + if (!d) return ''; + return d.slice(0, 10); + }
-
T YouTube @@ -272,10 +346,11 @@
- + - +
- {#if selectedVideo} + {#if activeTab === '🤖 Agent'} +
+

🤖 Agent Queue

+ {#if agentItems.length === 0} +

Queue empty. Watch a video, press n to write a note, check "Send to Agent", and save.

+ {:else} +
+ {#each agentItems as item} +
+
+
+

{item.title || 'Unknown'}

+

{item.channel_name}

+
+ PROMPT: +

{item.user_prompt}

+
+
+ + {item.status} + + {formatDate(item.created_at)} + ▶ Watch + {#if item.status === 'pending'} + + {/if} +
+
+
+
+ {/each} +
+ {/if} +
+ {:else if selectedVideo}
@@ -387,16 +541,26 @@ {#if activeNote === selectedVideo.video_id}
- - +
+ +
+ +
{:else if notes[selectedVideo.video_id]}