From d5af5e9b7d8ff091f5a35db50baac010b1655b40 Mon Sep 17 00:00:00 2001 From: "Gan, Jimmy" Date: Wed, 24 Jun 2026 02:38:04 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20rotate=2050=20channels=20daily=20?= =?UTF-8?q?=E2=80=94=20covers=20all=20229=20over=205=20days?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/fetcher.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/backend/fetcher.py b/backend/fetcher.py index d73f0e8..d3518bf 100644 --- a/backend/fetcher.py +++ b/backend/fetcher.py @@ -1,4 +1,5 @@ import asyncio +import os import html import logging import aiosqlite @@ -9,6 +10,33 @@ from db import DB_PATH log = logging.getLogger(__name__) +ROTATION_FILE = os.environ.get("ROTATION_FILE", os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "config", "rotation.txt")) + +def get_rotation_batch(channel_ids, batch_size=50): + """Read last position, return next batch of channels, save new position.""" + offset = 0 + try: + if os.path.exists(ROTATION_FILE): + with open(ROTATION_FILE) as f: + offset = int(f.read().strip()) + except: pass + + total = len(channel_ids) + if offset >= total: + offset = 0 + + end = min(offset + batch_size, total) + batch = channel_ids[offset:end] + + # Save next offset + new_offset = end if end < total else 0 + os.makedirs(os.path.dirname(ROTATION_FILE), exist_ok=True) + with open(ROTATION_FILE, 'w') as f: + f.write(str(new_offset)) + + log.info(f"Rotation: channels {offset+1}-{end} of {total} (next starts at {new_offset})") + return batch + async def fetch_subscriptions(): """Fetch latest videos from subscribed channels using YouTube Data API v3.""" try: @@ -48,10 +76,11 @@ async def fetch_subscriptions(): log.warning("No subscribed channels found. Make sure the authenticated account has subscriptions.") return 0 - # Step 2: For each channel, get latest videos + # Step 2: For each channel in today's rotation batch, get latest videos + batch = get_rotation_batch(channel_ids) new_count = 0 async with aiosqlite.connect(DB_PATH) as db: - for cid in channel_ids[:50]: + for cid in batch: try: req = youtube.search().list( part="snippet",