feat: use yt-dlp as primary transcript fetcher, fallback to youtube-transcript-api

- Add _ytdlp_transcript() — fetch transcripts via yt-dlp Python API
  (supports json3/srv3/vtt/srt subtitle formats, cookies.txt auth)
- Add _parse_subtitles() — parse subtitle formats into plain text
- Keep _yttranscript_api_transcript() as fallback
- download_transcript() tries yt-dlp first, then youtube-transcript-api
- Add yt-dlp to requirements.txt
- Update tests to mock both paths (all 23 pass)
- Document Google Data API v3 caption limitations in architecture.md
  (captions.list works, captions.download=403 for non-owned videos)

backend/summarizer.py

@@ -1,5 +1,6 @@
 import os
 import asyncio
+import json
 import logging
 import httpx
 import aiosqlite
@@ -41,10 +42,98 @@ for p in [os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__)
         COOKIES_PATH = p
         break
 
-def download_transcript(video_id: str) -> str:
+def _ytdlp_transcript(video_id: str) -> str | None:
+    """Fetch transcript using yt-dlp Python API (preferred)."""
+    try:
+        from yt_dlp import YoutubeDL
+
+        ydl_opts = {
+            "skip_download": True,
+            "writesubtitles": True,
+            "writeautomaticsub": True,
+            "subtitleslangs": ["en", "zh-Hans", "zh-Hant", "zh", "ja"],
+            "subtitlesformat": "json3",
+            "quiet": True,
+            "no_warnings": True,
+        }
+
+        proxy = proxy_from_env()
+        if proxy:
+            ydl_opts["proxy"] = proxy
+
+        if COOKIES_PATH and os.path.exists(COOKIES_PATH):
+            ydl_opts["cookiefile"] = COOKIES_PATH
+
+        with YoutubeDL(ydl_opts) as ydl:
+            info = ydl.extract_info(f"https://www.youtube.com/watch?v={video_id}", download=False)
+
+            # Try requested subtitles first (manually created), then auto-generated
+            subs = info.get("subtitles", {}) or {}
+            for lang in ["en", "zh-Hans", "zh-Hant", "zh", "ja"]:
+                if lang in subs:
+                    sub_data = subs[lang]
+                    for fmt in ["json3", "srv3", "srv2", "vtt", "srt"]:
+                        if fmt in sub_data:
+                            url = sub_data[fmt][0]["url"]
+                            resp = requests.get(url, timeout=30)
+                            if resp.status_code == 200:
+                                return _parse_subtitles(resp.text, fmt)
+
+            # Try auto-generated captions (requested_subs)
+            req_subs = info.get("requested_subtitles", {}) or {}
+            for lang in ["en", "zh-Hans", "zh-Hant", "zh", "ja"]:
+                if lang in req_subs:
+                    sub_data = req_subs[lang]
+                    for fmt in ["json3", "srv3", "srv2", "vtt", "srt"]:
+                        if fmt in sub_data:
+                            url = sub_data[fmt][0]["url"]
+                            resp = requests.get(url, timeout=30)
+                            if resp.status_code == 200:
+                                return _parse_subtitles(resp.text, fmt)
+
+        return None
+    except ImportError:
+        log.debug("yt-dlp not installed, skipping")
+        return None
+    except Exception as e:
+        log.warning(f"yt-dlp transcript failed for {video_id}: {e}")
+        return None
+
+
+def _parse_subtitles(text: str, fmt: str) -> str:
+    """Parse subtitle text from various formats into plain text."""
+    if fmt == "json3":
+        try:
+            data = json.loads(text)
+            events = data.get("events", [])
+            texts = []
+            for ev in events:
+                segs = ev.get("segs", [])
+                for seg in segs:
+                    txt = seg.get("utf8", "")
+                    if txt.strip():
+                        texts.append(txt.strip())
+            return " ".join(texts)
+        except (json.JSONDecodeError, KeyError):
+            pass
+    # Fallback: strip HTML-like tags from VTT/SRT
+    import re
+    # Remove timestamps and position markers
+    text = re.sub(r"^\d+:\d+:\d+[.,]\d+\s*-->\s*\d+:\d+:\d+[.,]\d+.*$", "", text, flags=re.MULTILINE)
+    text = re.sub(r"^\d+\s*$", "", text, flags=re.MULTILINE)
+    text = re.sub(r"<[^>]+>", "", text)
+    text = re.sub(r"&nbsp;", " ", text)
+    text = re.sub(r"&amp;", "&", text)
+    text = re.sub(r"&lt;", "<", text)
+    text = re.sub(r"&gt;", ">", text)
+    lines = [l.strip() for l in text.split("\n") if l.strip()]
+    return " ".join(lines)
+
+
+def _yttranscript_api_transcript(video_id: str) -> str | None:
+    """Fallback transcript fetcher using youtube-transcript-api."""
     try:
         session = requests.Session()
-        # Respect HTTP_PROXY/HTTPS_PROXY env vars
         proxy = proxy_from_env()
         if proxy:
             session.proxies = {"http": proxy, "https": proxy}
@@ -52,7 +141,7 @@ def download_transcript(video_id: str) -> str:
             "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
             "Accept-Language": "en-US,en;q=0.9"
         })
-        if COOKIES_PATH:
+        if COOKIES_PATH and os.path.exists(COOKIES_PATH):
             from http.cookiejar import MozillaCookieJar
             cj = MozillaCookieJar(COOKIES_PATH)
             cj.load(ignore_discard=True, ignore_expires=True)
@@ -68,9 +157,17 @@ def download_transcript(video_id: str) -> str:
         parts = transcript.fetch()
         return " ".join([part["text"] for part in parts])
     except Exception as e:
-        log.warning(f"Failed to fetch transcript for {video_id}: {e}")
+        log.warning(f"youtube-transcript-api failed for {video_id}: {e}")
         return None
 
+
+def download_transcript(video_id: str) -> str | None:
+    """Fetch transcript using yt-dlp first, fall back to youtube-transcript-api."""
+    transcript = _ytdlp_transcript(video_id)
+    if transcript:
+        return transcript
+    return _yttranscript_api_transcript(video_id)
+
 async def summarize_video(video_id: str) -> str | None:
     # 1. Try transcript, fall back to description (offload sync call to thread)
     transcript = await asyncio.to_thread(download_transcript, video_id)
@@ -141,10 +238,13 @@ async def summarize_video(video_id: str) -> str | None:
 
     return summary
 
-async def summarize_all_pending():
+async def summarize_all_pending(max_videos: int = 0):
     async with aiosqlite.connect(DB_PATH) as db:
         db.row_factory = aiosqlite.Row
-        cursor = await db.execute("SELECT video_id FROM videos WHERE summary IS NULL OR summary = ''")
+        if max_videos > 0:
+            cursor = await db.execute("SELECT video_id FROM videos WHERE summary IS NULL OR summary = '' LIMIT ?", (max_videos,))
+        else:
+            cursor = await db.execute("SELECT video_id FROM videos WHERE summary IS NULL OR summary = ''")
         rows = await cursor.fetchall()
 
     if not rows: