jimmy/t-youtube
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add FTS5 search endpoint + search param to /api/videos

Browse Source
This commit is contained in:
Gan, Jimmy
2026-07-09 22:47:54 +08:00
parent c5f3bd7696
commit 90a47c95c3
2 changed files with 211 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/db.py
+29 -1
View File
@@ -17,6 +17,7 @@ async def init_db():
thumbnail_url TEXT,
publish_date TEXT,
duration INTEGER,
view_count INTEGER DEFAULT 0,
transcript TEXT,
summary TEXT,
status TEXT DEFAULT 'pending',
@@ -39,10 +40,37 @@ async def init_db():
status TEXT DEFAULT 'pending',
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
processed_at TEXT,
result TEXT,
FOREIGN KEY (video_id) REFERENCES videos(video_id)
)
""")
# FTS5 virtual table for full-text search on title + summary
await db.execute("""
CREATE VIRTUAL TABLE IF NOT EXISTS videos_fts USING fts5(
title, summary,
content='videos',
content_rowid='rowid'
)
""")
# Trigger to keep FTS5 index in sync
await db.execute("""
CREATE TRIGGER IF NOT EXISTS videos_fts_insert AFTER INSERT ON videos
BEGIN
INSERT INTO videos_fts(rowid, title, summary) VALUES (new.rowid, new.title, new.summary);
END;
""")
await db.execute("""
CREATE TRIGGER IF NOT EXISTS videos_fts_update AFTER UPDATE ON videos
BEGIN
INSERT INTO videos_fts(videos_fts, rowid, title, summary) VALUES ('delete', old.rowid, old.title, old.summary);
INSERT INTO videos_fts(rowid, title, summary) VALUES (new.rowid, new.title, new.summary);
END;
""")
await db.execute("""
CREATE TRIGGER IF NOT EXISTS videos_fts_delete AFTER DELETE ON videos
BEGIN
INSERT INTO videos_fts(videos_fts, rowid, title, summary) VALUES ('delete', old.rowid, old.title, old.summary);
END;
""")
await db.commit()
async def get_db():
backend/main.py
+182 -5
View File
@@ -104,14 +104,49 @@ async def get_videos(
status: Optional[str] = "pending",
limit: int = 50,
channel: Optional[str] = None,
search: Optional[str] = None,
db: aiosqlite.Connection = Depends(get_db)
):
if channel:
query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
cursor = await db.execute(query, (status, channel, limit))
"""Get videos with optional search filter."""
if search and len(search) >= 2:
# Use search endpoint logic
q = search.replace('"', '\\"')
base_query = "SELECT v.* FROM videos_fts f JOIN videos v ON f.rowid = v.rowid WHERE videos_fts MATCH ?"
params = [q]
if status:
base_query += " AND v.status = ?"
params.append(status)
if channel:
base_query += " AND v.channel_name = ?"
params.append(channel)
base_query += " ORDER BY rank LIMIT ?"
params.append(limit)
else:
query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
cursor = await db.execute(query, (status, limit))
if channel:
query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
cursor = await db.execute(query, (status, channel, limit))
else:
query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
cursor = await db.execute(query, (status, limit))
rows = await cursor.fetchall()
return [dict(row) for row in rows]
try:
cursor = await db.execute(base_query, params)
except Exception:
# Fallback to LIKE search
like_query = "SELECT * FROM videos WHERE title LIKE ?"
params = [f"%{search}%"]
if status:
like_query += " AND status = ?"
params.append(status)
if channel:
like_query += " AND channel_name = ?"
params.append(channel)
like_query += " ORDER BY publish_date DESC LIMIT ?"
params.append(limit)
cursor = await db.execute(like_query, params)
rows = await cursor.fetchall()
return [dict(row) for row in rows]
@@ -281,6 +316,148 @@ async def remove_from_agent_queue(item_id: int, db: aiosqlite.Connection = Depen
await db.commit()
return {"message": f"Queue item {item_id} removed"}
# ---- Search (FTS5) ----
@app.get("/api/videos/search")
async def search_videos(
q: str = "",
status: Optional[str] = None,
limit: int = 50,
channel: Optional[str] = None,
db: aiosqlite.Connection = Depends(get_db)
):
"""Full-text search across video titles and summaries using FTS5."""
if not q or len(q) < 2:
return []
# Try FTS5 first
fts_query = q.replace('"', '\"')
base_query = "SELECT v.* FROM videos_fts f JOIN videos v ON f.rowid = v.rowid WHERE videos_fts MATCH ?"
params = [fts_query]
if status:
base_query += " AND v.status = ?"
params.append(status)
if channel:
base_query += " AND v.channel_name = ?"
params.append(channel)
base_query += " ORDER BY rank LIMIT ?"
params.append(limit)
try:
cursor = await db.execute(base_query, params)
except Exception:
# Fallback to LIKE if FTS5 not available
like_query = "SELECT * FROM videos WHERE title LIKE ?"
params = [f"%{q}%"]
if status:
like_query += " AND status = ?"
params.append(status)
if channel:
like_query += " AND channel_name = ?"
params.append(channel)
like_query += " LIMIT ?"
params.append(limit)
cursor = await db.execute(like_query, params)
rows = await cursor.fetchall()
return [dict(row) for row in rows]
# ---- Batch Operations ----
class BatchUpdate(BaseModel):
video_ids: list[str]
status: str
@app.post("/api/videos/batch")
async def batch_update(
batch: BatchUpdate,
db: aiosqlite.Connection = Depends(get_db)
):
"""Batch update video statuses."""
if not batch.video_ids:
return {"message": "No video IDs provided"}
if batch.status not in ["pending", "read", "skipped"]:
raise HTTPException(status_code=400, detail="Invalid status")
placeholders = ",".join(["?" for _ in batch.video_ids])
await db.execute(f"UPDATE videos SET status = ? WHERE video_id IN ({placeholders})",
[batch.status] + batch.video_ids)
await db.commit()
return {"message": f"Updated {len(batch.video_ids)} videos"}
# ---- Stats Dashboard ----
@app.get("/api/stats")
async def get_stats(db: aiosqlite.Connection = Depends(get_db)):
"""Return dashboard statistics."""
stats = {}
# Total videos
cursor = await db.execute("SELECT COUNT(*) FROM videos")
stats["total"] = (await cursor.fetchone())[0]
# Status breakdown
cursor = await db.execute("SELECT status, COUNT(*) FROM videos GROUP BY status")
stats["status_breakdown"] = {row[0]: row[1] for row in await cursor.fetchall()}
# Channels
cursor = await db.execute("SELECT COUNT(DISTINCT channel_name) FROM videos")
stats["channels"] = (await cursor.fetchone())[0]
# Videos with summaries vs without
cursor = await db.execute("SELECT COUNT(*) FROM videos WHERE summary IS NOT NULL AND summary != ''")
stats["with_summary"] = (await cursor.fetchone())[0]
# Recent activity (last 7 days)
cursor = await db.execute("SELECT COUNT(*) FROM videos WHERE created_at >= datetime('now', '-7 days')")
stats["last_7_days"] = (await cursor.fetchone())[0]
# Top channels by video count
cursor = await db.execute("SELECT channel_name, COUNT(*) as cnt FROM videos GROUP BY channel_name ORDER BY cnt DESC LIMIT 10")
stats["top_channels"] = [{"name": row[0], "count": row[1]} for row in await cursor.fetchall()]
# Summary rate
if stats["total"] > 0:
stats["summary_rate"] = round(stats["with_summary"] / stats["total"] * 100, 1)
else:
stats["summary_rate"] = 0
return stats
# ---- Duration + View Count Update ----
class VideoMetaUpdate(BaseModel):
duration: Optional[int] = None
view_count: Optional[int] = None
@app.patch("/api/videos/{video_id}/meta")
async def update_video_meta(
video_id: str,
meta: VideoMetaUpdate,
db: aiosqlite.Connection = Depends(get_db)
):
"""Update video metadata (duration, view count)."""
updates = []
params = []
if meta.duration is not None:
updates.append("duration = ?")
params.append(meta.duration)
if meta.view_count is not None:
updates.append("view_count = ?")
params.append(meta.view_count)
if not updates:
return {"message": "No updates provided"}
params.append(video_id)
await db.execute(f"UPDATE videos SET {', '.join(updates)} WHERE video_id = ?", params)
await db.commit()
return {"message": "Metadata updated"}
# ---- Include view_count in video list ----
# ---- Static frontend ----
frontend_build_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "frontend/dist")