feat: add FTS5 search endpoint + search param to /api/videos
This commit is contained in:
+29
-1
@@ -17,6 +17,7 @@ async def init_db():
|
|||||||
thumbnail_url TEXT,
|
thumbnail_url TEXT,
|
||||||
publish_date TEXT,
|
publish_date TEXT,
|
||||||
duration INTEGER,
|
duration INTEGER,
|
||||||
|
view_count INTEGER DEFAULT 0,
|
||||||
transcript TEXT,
|
transcript TEXT,
|
||||||
summary TEXT,
|
summary TEXT,
|
||||||
status TEXT DEFAULT 'pending',
|
status TEXT DEFAULT 'pending',
|
||||||
@@ -39,10 +40,37 @@ async def init_db():
|
|||||||
status TEXT DEFAULT 'pending',
|
status TEXT DEFAULT 'pending',
|
||||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||||
processed_at TEXT,
|
processed_at TEXT,
|
||||||
result TEXT,
|
|
||||||
FOREIGN KEY (video_id) REFERENCES videos(video_id)
|
FOREIGN KEY (video_id) REFERENCES videos(video_id)
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
|
# FTS5 virtual table for full-text search on title + summary
|
||||||
|
await db.execute("""
|
||||||
|
CREATE VIRTUAL TABLE IF NOT EXISTS videos_fts USING fts5(
|
||||||
|
title, summary,
|
||||||
|
content='videos',
|
||||||
|
content_rowid='rowid'
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
# Trigger to keep FTS5 index in sync
|
||||||
|
await db.execute("""
|
||||||
|
CREATE TRIGGER IF NOT EXISTS videos_fts_insert AFTER INSERT ON videos
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO videos_fts(rowid, title, summary) VALUES (new.rowid, new.title, new.summary);
|
||||||
|
END;
|
||||||
|
""")
|
||||||
|
await db.execute("""
|
||||||
|
CREATE TRIGGER IF NOT EXISTS videos_fts_update AFTER UPDATE ON videos
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO videos_fts(videos_fts, rowid, title, summary) VALUES ('delete', old.rowid, old.title, old.summary);
|
||||||
|
INSERT INTO videos_fts(rowid, title, summary) VALUES (new.rowid, new.title, new.summary);
|
||||||
|
END;
|
||||||
|
""")
|
||||||
|
await db.execute("""
|
||||||
|
CREATE TRIGGER IF NOT EXISTS videos_fts_delete AFTER DELETE ON videos
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO videos_fts(videos_fts, rowid, title, summary) VALUES ('delete', old.rowid, old.title, old.summary);
|
||||||
|
END;
|
||||||
|
""")
|
||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
async def get_db():
|
async def get_db():
|
||||||
|
|||||||
+177
@@ -104,8 +104,24 @@ async def get_videos(
|
|||||||
status: Optional[str] = "pending",
|
status: Optional[str] = "pending",
|
||||||
limit: int = 50,
|
limit: int = 50,
|
||||||
channel: Optional[str] = None,
|
channel: Optional[str] = None,
|
||||||
|
search: Optional[str] = None,
|
||||||
db: aiosqlite.Connection = Depends(get_db)
|
db: aiosqlite.Connection = Depends(get_db)
|
||||||
):
|
):
|
||||||
|
"""Get videos with optional search filter."""
|
||||||
|
if search and len(search) >= 2:
|
||||||
|
# Use search endpoint logic
|
||||||
|
q = search.replace('"', '\\"')
|
||||||
|
base_query = "SELECT v.* FROM videos_fts f JOIN videos v ON f.rowid = v.rowid WHERE videos_fts MATCH ?"
|
||||||
|
params = [q]
|
||||||
|
if status:
|
||||||
|
base_query += " AND v.status = ?"
|
||||||
|
params.append(status)
|
||||||
|
if channel:
|
||||||
|
base_query += " AND v.channel_name = ?"
|
||||||
|
params.append(channel)
|
||||||
|
base_query += " ORDER BY rank LIMIT ?"
|
||||||
|
params.append(limit)
|
||||||
|
else:
|
||||||
if channel:
|
if channel:
|
||||||
query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
|
query = "SELECT * FROM videos WHERE status = ? AND channel_name = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
|
||||||
cursor = await db.execute(query, (status, channel, limit))
|
cursor = await db.execute(query, (status, channel, limit))
|
||||||
@@ -115,6 +131,25 @@ async def get_videos(
|
|||||||
rows = await cursor.fetchall()
|
rows = await cursor.fetchall()
|
||||||
return [dict(row) for row in rows]
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
try:
|
||||||
|
cursor = await db.execute(base_query, params)
|
||||||
|
except Exception:
|
||||||
|
# Fallback to LIKE search
|
||||||
|
like_query = "SELECT * FROM videos WHERE title LIKE ?"
|
||||||
|
params = [f"%{search}%"]
|
||||||
|
if status:
|
||||||
|
like_query += " AND status = ?"
|
||||||
|
params.append(status)
|
||||||
|
if channel:
|
||||||
|
like_query += " AND channel_name = ?"
|
||||||
|
params.append(channel)
|
||||||
|
like_query += " ORDER BY publish_date DESC LIMIT ?"
|
||||||
|
params.append(limit)
|
||||||
|
cursor = await db.execute(like_query, params)
|
||||||
|
|
||||||
|
rows = await cursor.fetchall()
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
async def bg_fetch_and_summarize():
|
async def bg_fetch_and_summarize():
|
||||||
log.info("Starting background subscription fetch and summarization loop...")
|
log.info("Starting background subscription fetch and summarization loop...")
|
||||||
try:
|
try:
|
||||||
@@ -281,6 +316,148 @@ async def remove_from_agent_queue(item_id: int, db: aiosqlite.Connection = Depen
|
|||||||
await db.commit()
|
await db.commit()
|
||||||
return {"message": f"Queue item {item_id} removed"}
|
return {"message": f"Queue item {item_id} removed"}
|
||||||
|
|
||||||
|
# ---- Search (FTS5) ----
|
||||||
|
|
||||||
|
@app.get("/api/videos/search")
|
||||||
|
async def search_videos(
|
||||||
|
q: str = "",
|
||||||
|
status: Optional[str] = None,
|
||||||
|
limit: int = 50,
|
||||||
|
channel: Optional[str] = None,
|
||||||
|
db: aiosqlite.Connection = Depends(get_db)
|
||||||
|
):
|
||||||
|
"""Full-text search across video titles and summaries using FTS5."""
|
||||||
|
if not q or len(q) < 2:
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Try FTS5 first
|
||||||
|
fts_query = q.replace('"', '\"')
|
||||||
|
base_query = "SELECT v.* FROM videos_fts f JOIN videos v ON f.rowid = v.rowid WHERE videos_fts MATCH ?"
|
||||||
|
params = [fts_query]
|
||||||
|
|
||||||
|
if status:
|
||||||
|
base_query += " AND v.status = ?"
|
||||||
|
params.append(status)
|
||||||
|
if channel:
|
||||||
|
base_query += " AND v.channel_name = ?"
|
||||||
|
params.append(channel)
|
||||||
|
base_query += " ORDER BY rank LIMIT ?"
|
||||||
|
params.append(limit)
|
||||||
|
|
||||||
|
try:
|
||||||
|
cursor = await db.execute(base_query, params)
|
||||||
|
except Exception:
|
||||||
|
# Fallback to LIKE if FTS5 not available
|
||||||
|
like_query = "SELECT * FROM videos WHERE title LIKE ?"
|
||||||
|
params = [f"%{q}%"]
|
||||||
|
if status:
|
||||||
|
like_query += " AND status = ?"
|
||||||
|
params.append(status)
|
||||||
|
if channel:
|
||||||
|
like_query += " AND channel_name = ?"
|
||||||
|
params.append(channel)
|
||||||
|
like_query += " LIMIT ?"
|
||||||
|
params.append(limit)
|
||||||
|
cursor = await db.execute(like_query, params)
|
||||||
|
|
||||||
|
rows = await cursor.fetchall()
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
# ---- Batch Operations ----
|
||||||
|
|
||||||
|
class BatchUpdate(BaseModel):
|
||||||
|
video_ids: list[str]
|
||||||
|
status: str
|
||||||
|
|
||||||
|
@app.post("/api/videos/batch")
|
||||||
|
async def batch_update(
|
||||||
|
batch: BatchUpdate,
|
||||||
|
db: aiosqlite.Connection = Depends(get_db)
|
||||||
|
):
|
||||||
|
"""Batch update video statuses."""
|
||||||
|
if not batch.video_ids:
|
||||||
|
return {"message": "No video IDs provided"}
|
||||||
|
if batch.status not in ["pending", "read", "skipped"]:
|
||||||
|
raise HTTPException(status_code=400, detail="Invalid status")
|
||||||
|
|
||||||
|
placeholders = ",".join(["?" for _ in batch.video_ids])
|
||||||
|
await db.execute(f"UPDATE videos SET status = ? WHERE video_id IN ({placeholders})",
|
||||||
|
[batch.status] + batch.video_ids)
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
return {"message": f"Updated {len(batch.video_ids)} videos"}
|
||||||
|
|
||||||
|
# ---- Stats Dashboard ----
|
||||||
|
|
||||||
|
@app.get("/api/stats")
|
||||||
|
async def get_stats(db: aiosqlite.Connection = Depends(get_db)):
|
||||||
|
"""Return dashboard statistics."""
|
||||||
|
stats = {}
|
||||||
|
|
||||||
|
# Total videos
|
||||||
|
cursor = await db.execute("SELECT COUNT(*) FROM videos")
|
||||||
|
stats["total"] = (await cursor.fetchone())[0]
|
||||||
|
|
||||||
|
# Status breakdown
|
||||||
|
cursor = await db.execute("SELECT status, COUNT(*) FROM videos GROUP BY status")
|
||||||
|
stats["status_breakdown"] = {row[0]: row[1] for row in await cursor.fetchall()}
|
||||||
|
|
||||||
|
# Channels
|
||||||
|
cursor = await db.execute("SELECT COUNT(DISTINCT channel_name) FROM videos")
|
||||||
|
stats["channels"] = (await cursor.fetchone())[0]
|
||||||
|
|
||||||
|
# Videos with summaries vs without
|
||||||
|
cursor = await db.execute("SELECT COUNT(*) FROM videos WHERE summary IS NOT NULL AND summary != ''")
|
||||||
|
stats["with_summary"] = (await cursor.fetchone())[0]
|
||||||
|
|
||||||
|
# Recent activity (last 7 days)
|
||||||
|
cursor = await db.execute("SELECT COUNT(*) FROM videos WHERE created_at >= datetime('now', '-7 days')")
|
||||||
|
stats["last_7_days"] = (await cursor.fetchone())[0]
|
||||||
|
|
||||||
|
# Top channels by video count
|
||||||
|
cursor = await db.execute("SELECT channel_name, COUNT(*) as cnt FROM videos GROUP BY channel_name ORDER BY cnt DESC LIMIT 10")
|
||||||
|
stats["top_channels"] = [{"name": row[0], "count": row[1]} for row in await cursor.fetchall()]
|
||||||
|
|
||||||
|
# Summary rate
|
||||||
|
if stats["total"] > 0:
|
||||||
|
stats["summary_rate"] = round(stats["with_summary"] / stats["total"] * 100, 1)
|
||||||
|
else:
|
||||||
|
stats["summary_rate"] = 0
|
||||||
|
|
||||||
|
return stats
|
||||||
|
|
||||||
|
# ---- Duration + View Count Update ----
|
||||||
|
|
||||||
|
class VideoMetaUpdate(BaseModel):
|
||||||
|
duration: Optional[int] = None
|
||||||
|
view_count: Optional[int] = None
|
||||||
|
|
||||||
|
@app.patch("/api/videos/{video_id}/meta")
|
||||||
|
async def update_video_meta(
|
||||||
|
video_id: str,
|
||||||
|
meta: VideoMetaUpdate,
|
||||||
|
db: aiosqlite.Connection = Depends(get_db)
|
||||||
|
):
|
||||||
|
"""Update video metadata (duration, view count)."""
|
||||||
|
updates = []
|
||||||
|
params = []
|
||||||
|
if meta.duration is not None:
|
||||||
|
updates.append("duration = ?")
|
||||||
|
params.append(meta.duration)
|
||||||
|
if meta.view_count is not None:
|
||||||
|
updates.append("view_count = ?")
|
||||||
|
params.append(meta.view_count)
|
||||||
|
|
||||||
|
if not updates:
|
||||||
|
return {"message": "No updates provided"}
|
||||||
|
|
||||||
|
params.append(video_id)
|
||||||
|
await db.execute(f"UPDATE videos SET {', '.join(updates)} WHERE video_id = ?", params)
|
||||||
|
await db.commit()
|
||||||
|
return {"message": "Metadata updated"}
|
||||||
|
|
||||||
|
# ---- Include view_count in video list ----
|
||||||
|
|
||||||
# ---- Static frontend ----
|
# ---- Static frontend ----
|
||||||
|
|
||||||
frontend_build_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "frontend/dist")
|
frontend_build_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "frontend/dist")
|
||||||
|
|||||||
Reference in New Issue
Block a user