Gan, Jimmy
62 lines
1.9 KiB
Python
Executable File
62 lines
1.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import asyncio, os, sys, logging
|
|
|
|
REPO_ROOT = os.path.dirname(os.path.abspath(__file__))
|
|
BACKEND = os.path.join(REPO_ROOT, "backend")
|
|
ENV_PATH = os.path.join(REPO_ROOT, ".env")
|
|
if os.path.exists(ENV_PATH):
|
|
with open(ENV_PATH) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line and not line.startswith("#") and "=" in line:
|
|
key, _, value = line.partition("=")
|
|
value = value.strip().strip("'\"")
|
|
os.environ[key] = value
|
|
|
|
sys.path.insert(0, BACKEND)
|
|
os.chdir(BACKEND)
|
|
|
|
from db import init_db, DB_PATH
|
|
from fetcher import fetch_subscriptions
|
|
from summarizer import summarize_video
|
|
import aiosqlite
|
|
|
|
BATCH_SIZE = int(os.environ.get("SUMMARIZE_BATCH", "20"))
|
|
BATCH_DELAY = float(os.environ.get("SUMMARIZE_BATCH_DELAY", "2.0"))
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
|
|
async def main():
|
|
await init_db()
|
|
|
|
count = await fetch_subscriptions()
|
|
print(f"Fetched {count} new videos")
|
|
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
cur = await db.execute("SELECT COUNT(*) FROM videos WHERE summary IS NULL OR summary = ''")
|
|
total = (await cur.fetchone())[0]
|
|
|
|
if total == 0:
|
|
print("No pending videos to summarize.")
|
|
return
|
|
|
|
print(f"Processing batch of {BATCH_SIZE} (of {total} pending)...")
|
|
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
cur = await db.execute("SELECT video_id FROM videos WHERE summary IS NULL OR summary = '' LIMIT ?", (BATCH_SIZE,))
|
|
rows = await cur.fetchall()
|
|
|
|
done = 0
|
|
for row in rows:
|
|
await summarize_video(row["video_id"])
|
|
done += 1
|
|
if done < len(rows):
|
|
await asyncio.sleep(BATCH_DELAY)
|
|
|
|
remaining = total - done
|
|
print(f"Batch done: {done}/{len(rows)}, {remaining} pending")
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|