From 90a47c95c31a4ec6fdb6ef5b02de66f66ab229f1 Mon Sep 17 00:00:00 2001 From: "Gan, Jimmy" Date: Thu, 9 Jul 2026 22:47:54 +0800 Subject: [PATCH] feat: add FTS5 search endpoint + search param to /api/videos --- backend/db.py | 30 +++++++- backend/main.py | 187 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 211 insertions(+), 6 deletions(-) diff --git a/backend/db.py b/backend/db.py index 7f81d55..d84da04 100644 --- a/backend/db.py +++ b/backend/db.py @@ -17,6 +17,7 @@ async def init_db(): thumbnail_url TEXT, publish_date TEXT, duration INTEGER, + view_count INTEGER DEFAULT 0, transcript TEXT, summary TEXT, status TEXT DEFAULT 'pending', @@ -39,10 +40,37 @@ async def init_db(): status TEXT DEFAULT 'pending', created_at TEXT DEFAULT CURRENT_TIMESTAMP, processed_at TEXT, - result TEXT, FOREIGN KEY (video_id) REFERENCES videos(video_id) ) """) + # FTS5 virtual table for full-text search on title + summary + await db.execute(""" + CREATE VIRTUAL TABLE IF NOT EXISTS videos_fts USING fts5( + title, summary, + content='videos', + content_rowid='rowid' + ) + """) + # Trigger to keep FTS5 index in sync + await db.execute(""" + CREATE TRIGGER IF NOT EXISTS videos_fts_insert AFTER INSERT ON videos + BEGIN + INSERT INTO videos_fts(rowid, title, summary) VALUES (new.rowid, new.title, new.summary); + END; + """) + await db.execute(""" + CREATE TRIGGER IF NOT EXISTS videos_fts_update AFTER UPDATE ON videos + BEGIN + INSERT INTO videos_fts(videos_fts, rowid, title, summary) VALUES ('delete', old.rowid, old.title, old.summary); + INSERT INTO videos_fts(rowid, title, summary) VALUES (new.rowid, new.title, new.summary); + END; + """) + await db.execute(""" + CREATE TRIGGER IF NOT EXISTS videos_fts_delete AFTER DELETE ON videos + BEGIN + INSERT INTO videos_fts(videos_fts, rowid, title, summary) VALUES ('delete', old.rowid, old.title, old.summary); + END; + """) await db.commit() async def get_db(): diff --git a/backend/main.py b/backend/main.py index ad4af08..5cf1cd1 100644 --- a/backend/main.py +++ b/backend/main.py @@ -104,14 +104,49 @@ async def get_videos( status: Optional[str] = "pending", limit: int = 50, channel: Optional[str] = None, + search: Optional[str] = None, db: aiosqlite.Connection = Depends(get_db) ): - if channel: - query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?" - cursor = await db.execute(query, (status, channel, limit)) + """Get videos with optional search filter.""" + if search and len(search) >= 2: + # Use search endpoint logic + q = search.replace('"', '\\"') + base_query = "SELECT v.* FROM videos_fts f JOIN videos v ON f.rowid = v.rowid WHERE videos_fts MATCH ?" + params = [q] + if status: + base_query += " AND v.status = ?" + params.append(status) + if channel: + base_query += " AND v.channel_name = ?" + params.append(channel) + base_query += " ORDER BY rank LIMIT ?" + params.append(limit) else: - query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?" - cursor = await db.execute(query, (status, limit)) + if channel: + query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?" + cursor = await db.execute(query, (status, channel, limit)) + else: + query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?" + cursor = await db.execute(query, (status, limit)) + rows = await cursor.fetchall() + return [dict(row) for row in rows] + + try: + cursor = await db.execute(base_query, params) + except Exception: + # Fallback to LIKE search + like_query = "SELECT * FROM videos WHERE title LIKE ?" + params = [f"%{search}%"] + if status: + like_query += " AND status = ?" + params.append(status) + if channel: + like_query += " AND channel_name = ?" + params.append(channel) + like_query += " ORDER BY publish_date DESC LIMIT ?" + params.append(limit) + cursor = await db.execute(like_query, params) + rows = await cursor.fetchall() return [dict(row) for row in rows] @@ -281,6 +316,148 @@ async def remove_from_agent_queue(item_id: int, db: aiosqlite.Connection = Depen await db.commit() return {"message": f"Queue item {item_id} removed"} +# ---- Search (FTS5) ---- + +@app.get("/api/videos/search") +async def search_videos( + q: str = "", + status: Optional[str] = None, + limit: int = 50, + channel: Optional[str] = None, + db: aiosqlite.Connection = Depends(get_db) +): + """Full-text search across video titles and summaries using FTS5.""" + if not q or len(q) < 2: + return [] + + # Try FTS5 first + fts_query = q.replace('"', '\"') + base_query = "SELECT v.* FROM videos_fts f JOIN videos v ON f.rowid = v.rowid WHERE videos_fts MATCH ?" + params = [fts_query] + + if status: + base_query += " AND v.status = ?" + params.append(status) + if channel: + base_query += " AND v.channel_name = ?" + params.append(channel) + base_query += " ORDER BY rank LIMIT ?" + params.append(limit) + + try: + cursor = await db.execute(base_query, params) + except Exception: + # Fallback to LIKE if FTS5 not available + like_query = "SELECT * FROM videos WHERE title LIKE ?" + params = [f"%{q}%"] + if status: + like_query += " AND status = ?" + params.append(status) + if channel: + like_query += " AND channel_name = ?" + params.append(channel) + like_query += " LIMIT ?" + params.append(limit) + cursor = await db.execute(like_query, params) + + rows = await cursor.fetchall() + return [dict(row) for row in rows] + +# ---- Batch Operations ---- + +class BatchUpdate(BaseModel): + video_ids: list[str] + status: str + +@app.post("/api/videos/batch") +async def batch_update( + batch: BatchUpdate, + db: aiosqlite.Connection = Depends(get_db) +): + """Batch update video statuses.""" + if not batch.video_ids: + return {"message": "No video IDs provided"} + if batch.status not in ["pending", "read", "skipped"]: + raise HTTPException(status_code=400, detail="Invalid status") + + placeholders = ",".join(["?" for _ in batch.video_ids]) + await db.execute(f"UPDATE videos SET status = ? WHERE video_id IN ({placeholders})", + [batch.status] + batch.video_ids) + await db.commit() + + return {"message": f"Updated {len(batch.video_ids)} videos"} + +# ---- Stats Dashboard ---- + +@app.get("/api/stats") +async def get_stats(db: aiosqlite.Connection = Depends(get_db)): + """Return dashboard statistics.""" + stats = {} + + # Total videos + cursor = await db.execute("SELECT COUNT(*) FROM videos") + stats["total"] = (await cursor.fetchone())[0] + + # Status breakdown + cursor = await db.execute("SELECT status, COUNT(*) FROM videos GROUP BY status") + stats["status_breakdown"] = {row[0]: row[1] for row in await cursor.fetchall()} + + # Channels + cursor = await db.execute("SELECT COUNT(DISTINCT channel_name) FROM videos") + stats["channels"] = (await cursor.fetchone())[0] + + # Videos with summaries vs without + cursor = await db.execute("SELECT COUNT(*) FROM videos WHERE summary IS NOT NULL AND summary != ''") + stats["with_summary"] = (await cursor.fetchone())[0] + + # Recent activity (last 7 days) + cursor = await db.execute("SELECT COUNT(*) FROM videos WHERE created_at >= datetime('now', '-7 days')") + stats["last_7_days"] = (await cursor.fetchone())[0] + + # Top channels by video count + cursor = await db.execute("SELECT channel_name, COUNT(*) as cnt FROM videos GROUP BY channel_name ORDER BY cnt DESC LIMIT 10") + stats["top_channels"] = [{"name": row[0], "count": row[1]} for row in await cursor.fetchall()] + + # Summary rate + if stats["total"] > 0: + stats["summary_rate"] = round(stats["with_summary"] / stats["total"] * 100, 1) + else: + stats["summary_rate"] = 0 + + return stats + +# ---- Duration + View Count Update ---- + +class VideoMetaUpdate(BaseModel): + duration: Optional[int] = None + view_count: Optional[int] = None + +@app.patch("/api/videos/{video_id}/meta") +async def update_video_meta( + video_id: str, + meta: VideoMetaUpdate, + db: aiosqlite.Connection = Depends(get_db) +): + """Update video metadata (duration, view count).""" + updates = [] + params = [] + if meta.duration is not None: + updates.append("duration = ?") + params.append(meta.duration) + if meta.view_count is not None: + updates.append("view_count = ?") + params.append(meta.view_count) + + if not updates: + return {"message": "No updates provided"} + + params.append(video_id) + await db.execute(f"UPDATE videos SET {', '.join(updates)} WHERE video_id = ?", params) + await db.commit() + return {"message": "Metadata updated"} + +# ---- Include view_count in video list ---- + # ---- Static frontend ---- frontend_build_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "frontend/dist")