sprint: fixes batch — saveNote history, DeepSeek retry, channel rotation, test fixes, docker env, blocking transcript, sync errors

backend/fetcher.py

@@ -1,5 +1,6 @@
 import asyncio
 import os
+import json
 import html
 import logging
 import aiosqlite
@@ -14,6 +15,36 @@ log = logging.getLogger(__name__)
 
 ATOM_NS = "http://www.w3.org/2005/Atom"
 
+CHANNELS_PER_SYNC = 50  # Rotate through all channels — N per sync
+_ROTATION_FILE = os.path.join(os.environ.get("DB_DIR", os.path.dirname(os.path.abspath(__file__))), "sync_rotation.json")
+
+
+def _get_rotation_slice(total: int) -> tuple[int, int]:
+    """Return (start, end) indices for this sync's channel slice."""
+    state = {"offset": 0, "total": total}
+    if os.path.exists(_ROTATION_FILE):
+        try:
+            with open(_ROTATION_FILE) as f:
+                state = json.load(f)
+        except (json.JSONDecodeError, OSError):
+            pass
+    # If total changed (subscribed/unsubscribed), reset
+    if state.get("total") != total:
+        state = {"offset": 0, "total": total}
+    start = state["offset"]
+    end = min(start + CHANNELS_PER_SYNC, total)
+    return start, end
+
+
+def _save_rotation_slice(total: int, start: int, end: int):
+    """Persist next sync offset."""
+    next_offset = end  # Continue from where we left off
+    if next_offset >= total:
+        next_offset = 0  # Wrap around — full cycle complete
+    os.makedirs(os.path.dirname(_ROTATION_FILE), exist_ok=True)
+    with open(_ROTATION_FILE, "w") as f:
+        json.dump({"offset": next_offset, "total": total}, f)
+
 async def _fetch_rss(client, channel_id):
     """Fetch latest videos from a channel's RSS feed (free). Returns list of (video_id, title)."""
     url = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
@@ -61,12 +92,17 @@ async def fetch_subscriptions():
 
     log.info(f"Found {len(channel_ids)} subscribed channels")
 
-    # Step 2: Fetch RSS from all channels (free, parallel)
+    # Step 2: Rotate — only process CHANNELS_PER_SYNC channels per sync
+    start, end = _get_rotation_slice(len(channel_ids))
+    slice_ids = channel_ids[start:end]
+    log.info(f"Processing channels {start+1}–{end} of {len(channel_ids)} ({len(slice_ids)} this sync)")
+
+    # Step 3: Fetch RSS from slice channels (free, parallel)
     rss_videos = {}
     async with httpx.AsyncClient(timeout=20) as client:
-        tasks = [_fetch_rss(client, cid) for cid in channel_ids]
+        tasks = [_fetch_rss(client, cid) for cid in slice_ids]
         results = await asyncio.gather(*tasks)
-        for cid, entries in zip(channel_ids, results):
+        for cid, entries in zip(slice_ids, results):
             if entries:
                 rss_videos[cid] = entries
 
@@ -81,6 +117,7 @@ async def fetch_subscriptions():
     log.info(f"RSS discovered {len(all_video_ids)} videos from {len(rss_videos)} channels")
 
     if not all_video_ids:
+        _save_rotation_slice(len(channel_ids), start, end)
         return 0
 
     # Step 3: Check which videos are new (in SQLite)
@@ -94,6 +131,7 @@ async def fetch_subscriptions():
     log.info(f"{len(new_ids)} new videos to fetch metadata for")
 
     if not new_ids:
+        _save_rotation_slice(len(channel_ids), start, end)
         return 0
 
     # Step 4: Fetch metadata via videos.list (1 unit per 50 videos)
@@ -129,6 +167,10 @@ async def fetch_subscriptions():
         await db.commit()
 
     log.info(f"Ingestion completed. Found {new_count} new videos.")
+
+    # Save rotation state for next sync
+    _save_rotation_slice(len(channel_ids), start, end)
+
     return new_count