Initial commit: Add T YouTube text-based reader web app

- FastAPI backend running on port 8000 - Svelte 5 runes frontend dashboard supporting Read/Skip state toggles - YouTube subscription feed scraper using yt-dlp & Netscape cookies.txt - Transcript downloader and Claude 3.5 Haiku summarizer via bytecatcode proxy - Multi-stage Dockerfile and docker-compose.yml configuration Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-03 11:00:34 +08:00
commit dbd3dc0237
17 changed files with 967 additions and 0 deletions
@@ -0,0 +1,96 @@
+import asyncio
+import json
+import os
+import logging
+import aiosqlite
+from db import DB_PATH, init_db
+
+log = logging.getLogger(__name__)
+
+COOKIES_PATH = os.environ.get("COOKIES_PATH", "/app/config/cookies.txt")
+if not os.path.exists(COOKIES_PATH):
+    # Fallback to local directory
+    COOKIES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cookies.txt")
+
+async def fetch_subscriptions():
+    if not os.path.exists(COOKIES_PATH):
+        log.error(f"cookies.txt not found at {COOKIES_PATH}. Ingestion aborted.")
+        return 0
+
+    log.info(f"Fetching subscriptions using cookies at {COOKIES_PATH}")
+
+    cmd = [
+        "yt-dlp",
+        "--flat-playlist",
+        "--dump-json",
+        "--cookies", COOKIES_PATH,
+        "--playlist-end", "40",
+        "https://www.youtube.com/feed/subscriptions"
+    ]
+
+    try:
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+        stdout, stderr = await process.communicate()
+    except Exception as e:
+        log.error(f"Failed to run yt-dlp: {e}")
+        return 0
+
+    if process.returncode != 0:
+        log.error(f"yt-dlp error output: {stderr.decode('utf-8', errors='ignore')}")
+        return 0
+
+    lines = stdout.decode('utf-8', errors='ignore').strip().split('\n')
+    new_count = 0
+
+    async with aiosqlite.connect(DB_PATH) as db:
+        for line in lines:
+            if not line:
+                continue
+            try:
+                video = json.loads(line)
+                video_id = video.get("id")
+                if not video_id:
+                    continue
+
+                title = video.get("title", "Untitled")
+                channel_name = video.get("uploader") or video.get("channel") or "Unknown Channel"
+                channel_id = video.get("uploader_id") or video.get("channel_id")
+                url = f"https://www.youtube.com/watch?v={video_id}"
+                thumbnail_url = video.get("thumbnail") or f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
+
+                raw_date = video.get("upload_date")
+                if raw_date and len(raw_date) == 8:
+                    publish_date = f"{raw_date[:4]}-{raw_date[4:6]}-{raw_date[6:]}"
+                else:
+                    publish_date = None
+
+                duration = video.get("duration")
+
+                # Check if video already exists in db
+                cursor = await db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,))
+                exists = await cursor.fetchone()
+                if not exists:
+                    await db.execute("""
+                        INSERT INTO videos (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration, status)
+                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending')
+                    """, (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration))
+                    new_count += 1
+            except Exception as e:
+                log.warning(f"Error parsing yt-dlp line: {e}")
+                continue
+        await db.commit()
+
+    log.info(f"Ingestion completed. Found {new_count} new videos.")
+    return new_count
+
+if __name__ == "__main__":
+    import sys
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+    async def main():
+        await init_db()
+        await fetch_subscriptions()
+    asyncio.run(main())