import asyncio import os import html import logging import aiosqlite from googleapiclient.discovery import build from googleapiclient.errors import HttpError from auth import get_credentials from db import DB_PATH log = logging.getLogger(__name__) ROTATION_FILE = os.environ.get("ROTATION_FILE", os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "config", "rotation.txt")) def get_rotation_batch(channel_ids, batch_size=50): """Read last position, return next batch of channels, save new position.""" offset = 0 try: if os.path.exists(ROTATION_FILE): with open(ROTATION_FILE) as f: offset = int(f.read().strip()) except: pass total = len(channel_ids) if offset >= total: offset = 0 end = min(offset + batch_size, total) batch = channel_ids[offset:end] # Save next offset new_offset = end if end < total else 0 os.makedirs(os.path.dirname(ROTATION_FILE), exist_ok=True) with open(ROTATION_FILE, 'w') as f: f.write(str(new_offset)) log.info(f"Rotation: channels {offset+1}-{end} of {total} (next starts at {new_offset})") return batch async def fetch_subscriptions(): """Fetch latest videos from subscribed channels using YouTube Data API v3.""" try: creds = get_credentials() except FileNotFoundError as e: log.error(f"OAuth setup incomplete: {e}") return 0 except Exception as e: log.error(f"OAuth credential error: {e}") return 0 youtube = build("youtube", "v3", credentials=creds) # Step 1: Get subscribed channel IDs channel_ids = [] try: request = youtube.subscriptions().list( part="snippet", mine=True, maxResults=50, order="alphabetical" ) while request: response = request.execute() for item in response.get("items", []): cid = item["snippet"]["resourceId"]["channelId"] if cid not in channel_ids: channel_ids.append(cid) request = youtube.subscriptions().list_next(request, response) except HttpError as e: log.error(f"YouTube API error fetching subscriptions: {e}") return 0 log.info(f"Found {len(channel_ids)} subscribed channels") if not channel_ids: log.warning("No subscribed channels found. Make sure the authenticated account has subscriptions.") return 0 # Step 2: For each channel in today's rotation batch, get latest videos batch = get_rotation_batch(channel_ids) new_count = 0 async with aiosqlite.connect(DB_PATH) as db: for cid in batch: try: req = youtube.search().list( part="snippet", channelId=cid, maxResults=10, order="date", type="video" ) resp = req.execute() for item in resp.get("items", []): vid = item["id"]["videoId"] snippet = item["snippet"] cursor = await db.execute( "SELECT 1 FROM videos WHERE video_id = ?", (vid,) ) if await cursor.fetchone(): continue pub_time = snippet.get("publishTime", "") await db.execute(""" INSERT INTO videos (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date) VALUES (?, ?, ?, ?, ?, ?, ?) """, ( vid, html.unescape(snippet.get("title", "Untitled")), html.unescape(snippet.get("channelTitle", "Unknown")), snippet.get("channelId", cid), f"https://www.youtube.com/watch?v={vid}", snippet.get("thumbnails", {}).get("high", {}).get("url", ""), pub_time[:10] if pub_time else None )) new_count += 1 log.debug(f"New video: {snippet.get('title', 'Untitled')[:50]}") except HttpError as e: log.warning(f"Channel {cid} fetch error: {e}") continue await db.commit() log.info(f"Ingestion completed. Found {new_count} new videos.") return new_count if __name__ == "__main__": import sys logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') async def main(): from db import init_db await init_db() count = await fetch_subscriptions() print(f"\nDone. {count} new videos ingested.") asyncio.run(main())