remove unused ffmpeg from Dockerfile (saves ~450MB)
This commit is contained in:
@@ -13,7 +13,6 @@ WORKDIR /app
|
|||||||
# Install system dependencies
|
# Install system dependencies
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
curl \
|
curl \
|
||||||
ffmpeg \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+34
-5
@@ -9,7 +9,7 @@ if os.path.exists(ENV_PATH):
|
|||||||
line = line.strip()
|
line = line.strip()
|
||||||
if line and not line.startswith("#") and "=" in line:
|
if line and not line.startswith("#") and "=" in line:
|
||||||
key, _, value = line.partition("=")
|
key, _, value = line.partition("=")
|
||||||
value = value.strip().strip("'\"")
|
value = value.strip().strip("'\\"")
|
||||||
os.environ[key] = value
|
os.environ[key] = value
|
||||||
|
|
||||||
sys.path.insert(0, REPO_ROOT)
|
sys.path.insert(0, REPO_ROOT)
|
||||||
@@ -18,17 +18,46 @@ sys.path.insert(0, os.path.join(REPO_ROOT, "backend"))
|
|||||||
from backend.db import init_db
|
from backend.db import init_db
|
||||||
from backend.fetcher import fetch_subscriptions
|
from backend.fetcher import fetch_subscriptions
|
||||||
from backend.summarizer import summarize_all_pending
|
from backend.summarizer import summarize_all_pending
|
||||||
|
import aiosqlite
|
||||||
|
|
||||||
|
BATCH_SIZE = int(os.environ.get("SUMMARIZE_BATCH", "50"))
|
||||||
|
BATCH_DELAY = float(os.environ.get("SUMMARIZE_BATCH_DELAY", "1.0"))
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||||
|
|
||||||
async def main():
|
async def main():
|
||||||
await init_db()
|
await init_db()
|
||||||
|
|
||||||
|
# Fetch new videos first
|
||||||
count = await fetch_subscriptions()
|
count = await fetch_subscriptions()
|
||||||
print(f"Fetched {count} new videos")
|
print(f"Fetched {count} new videos")
|
||||||
if count > 0:
|
|
||||||
print(f"Summarizing {count} videos...")
|
# Count pending
|
||||||
await summarize_all_pending()
|
async with aiosqlite.connect(os.path.join(REPO_ROOT, "backend", "data", "videos.db")) as db:
|
||||||
print("Done")
|
cur = await db.execute("SELECT COUNT(*) FROM videos WHERE summary IS NULL OR summary = ''")
|
||||||
|
total = (await cur.fetchone())[0]
|
||||||
|
|
||||||
|
if total == 0:
|
||||||
|
print("No pending videos to summarize.")
|
||||||
|
return
|
||||||
|
|
||||||
|
print(f"Processing batch of up to {BATCH_SIZE} videos (of {total} pending)...")
|
||||||
|
|
||||||
|
# Get pending video IDs
|
||||||
|
async with aiosqlite.connect(os.path.join(REPO_ROOT, "backend", "data", "videos.db")) as db:
|
||||||
|
db.row_factory = aiosqlite.Row
|
||||||
|
cur = await db.execute("SELECT video_id FROM videos WHERE summary IS NULL OR summary = '' LIMIT ?", (BATCH_SIZE,))
|
||||||
|
rows = await cur.fetchall()
|
||||||
|
|
||||||
|
from backend.summarizer import summarize_video
|
||||||
|
done = 0
|
||||||
|
for row in rows:
|
||||||
|
await summarize_video(row["video_id"])
|
||||||
|
done += 1
|
||||||
|
if done < len(rows):
|
||||||
|
await asyncio.sleep(BATCH_DELAY)
|
||||||
|
|
||||||
|
print(f"Processed {done}/{len(rows)} in batch, {total - done} remaining")
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
asyncio.run(main())
|
||||||
|
|||||||
Reference in New Issue
Block a user