#!/usr/bin/env python3 import asyncio, os, sys, logging REPO_ROOT = os.path.dirname(os.path.abspath(__file__)) ENV_PATH = os.path.join(REPO_ROOT, ".env") if os.path.exists(ENV_PATH): with open(ENV_PATH) as f: for line in f: line = line.strip() if line and not line.startswith("#") and "=" in line: key, _, value = line.partition("=") value = value.strip().strip("'\\"") os.environ[key] = value sys.path.insert(0, REPO_ROOT) sys.path.insert(0, os.path.join(REPO_ROOT, "backend")) from backend.db import init_db from backend.fetcher import fetch_subscriptions from backend.summarizer import summarize_all_pending import aiosqlite BATCH_SIZE = int(os.environ.get("SUMMARIZE_BATCH", "50")) BATCH_DELAY = float(os.environ.get("SUMMARIZE_BATCH_DELAY", "1.0")) logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") async def main(): await init_db() # Fetch new videos first count = await fetch_subscriptions() print(f"Fetched {count} new videos") # Count pending async with aiosqlite.connect(os.path.join(REPO_ROOT, "backend", "data", "videos.db")) as db: cur = await db.execute("SELECT COUNT(*) FROM videos WHERE summary IS NULL OR summary = ''") total = (await cur.fetchone())[0] if total == 0: print("No pending videos to summarize.") return print(f"Processing batch of up to {BATCH_SIZE} videos (of {total} pending)...") # Get pending video IDs async with aiosqlite.connect(os.path.join(REPO_ROOT, "backend", "data", "videos.db")) as db: db.row_factory = aiosqlite.Row cur = await db.execute("SELECT video_id FROM videos WHERE summary IS NULL OR summary = '' LIMIT ?", (BATCH_SIZE,)) rows = await cur.fetchall() from backend.summarizer import summarize_video done = 0 for row in rows: await summarize_video(row["video_id"]) done += 1 if done < len(rows): await asyncio.sleep(BATCH_DELAY) print(f"Processed {done}/{len(rows)} in batch, {total - done} remaining") if __name__ == "__main__": asyncio.run(main())