diff --git a/backend/fetcher.py b/backend/fetcher.py index d73f0e8..d3518bf 100644 --- a/backend/fetcher.py +++ b/backend/fetcher.py @@ -1,4 +1,5 @@ import asyncio +import os import html import logging import aiosqlite @@ -9,6 +10,33 @@ from db import DB_PATH log = logging.getLogger(__name__) +ROTATION_FILE = os.environ.get("ROTATION_FILE", os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "config", "rotation.txt")) + +def get_rotation_batch(channel_ids, batch_size=50): + """Read last position, return next batch of channels, save new position.""" + offset = 0 + try: + if os.path.exists(ROTATION_FILE): + with open(ROTATION_FILE) as f: + offset = int(f.read().strip()) + except: pass + + total = len(channel_ids) + if offset >= total: + offset = 0 + + end = min(offset + batch_size, total) + batch = channel_ids[offset:end] + + # Save next offset + new_offset = end if end < total else 0 + os.makedirs(os.path.dirname(ROTATION_FILE), exist_ok=True) + with open(ROTATION_FILE, 'w') as f: + f.write(str(new_offset)) + + log.info(f"Rotation: channels {offset+1}-{end} of {total} (next starts at {new_offset})") + return batch + async def fetch_subscriptions(): """Fetch latest videos from subscribed channels using YouTube Data API v3.""" try: @@ -48,10 +76,11 @@ async def fetch_subscriptions(): log.warning("No subscribed channels found. Make sure the authenticated account has subscriptions.") return 0 - # Step 2: For each channel, get latest videos + # Step 2: For each channel in today's rotation batch, get latest videos + batch = get_rotation_batch(channel_ids) new_count = 0 async with aiosqlite.connect(DB_PATH) as db: - for cid in channel_ids[:50]: + for cid in batch: try: req = youtube.search().list( part="snippet",