import asyncio import json import os import logging import aiosqlite from db import DB_PATH, init_db log = logging.getLogger(__name__) COOKIES_PATH = os.environ.get("COOKIES_PATH", "/app/config/cookies.txt") if not os.path.exists(COOKIES_PATH): # Fallback to local directory COOKIES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cookies.txt") async def fetch_subscriptions(): if not os.path.exists(COOKIES_PATH): log.error(f"cookies.txt not found at {COOKIES_PATH}. Ingestion aborted.") return 0 log.info(f"Fetching subscriptions using cookies at {COOKIES_PATH}") cmd = [ "yt-dlp", "--flat-playlist", "--dump-json", "--cookies", COOKIES_PATH, "--playlist-end", "40", "https://www.youtube.com/feed/subscriptions" ] try: process = await asyncio.create_subprocess_exec( *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE ) stdout, stderr = await process.communicate() except Exception as e: log.error(f"Failed to run yt-dlp: {e}") return 0 if process.returncode != 0: log.error(f"yt-dlp error output: {stderr.decode('utf-8', errors='ignore')}") return 0 lines = stdout.decode('utf-8', errors='ignore').strip().split('\n') new_count = 0 async with aiosqlite.connect(DB_PATH) as db: for line in lines: if not line: continue try: video = json.loads(line) video_id = video.get("id") if not video_id: continue title = video.get("title", "Untitled") channel_name = video.get("uploader") or video.get("channel") or "Unknown Channel" channel_id = video.get("uploader_id") or video.get("channel_id") url = f"https://www.youtube.com/watch?v={video_id}" thumbnail_url = video.get("thumbnail") or f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg" raw_date = video.get("upload_date") if raw_date and len(raw_date) == 8: publish_date = f"{raw_date[:4]}-{raw_date[4:6]}-{raw_date[6:]}" else: publish_date = None duration = video.get("duration") # Check if video already exists in db cursor = await db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,)) exists = await cursor.fetchone() if not exists: await db.execute(""" INSERT INTO videos (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending') """, (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration)) new_count += 1 except Exception as e: log.warning(f"Error parsing yt-dlp line: {e}") continue await db.commit() log.info(f"Ingestion completed. Found {new_count} new videos.") return new_count if __name__ == "__main__": import sys logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') async def main(): await init_db() await fetch_subscriptions() asyncio.run(main())