Gan, Jimmy
98 lines
3.5 KiB
Python
98 lines
3.5 KiB
Python
import asyncio
|
|
import json
|
|
import os
|
|
import logging
|
|
import aiosqlite
|
|
from db import DB_PATH, init_db
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
COOKIES_PATH = os.environ.get("COOKIES_PATH", "/app/config/cookies.txt")
|
|
if not os.path.exists(COOKIES_PATH):
|
|
# Fallback to local directory
|
|
COOKIES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cookies.txt")
|
|
|
|
async def fetch_subscriptions():
|
|
if not os.path.exists(COOKIES_PATH):
|
|
log.error(f"cookies.txt not found at {COOKIES_PATH}. Ingestion aborted.")
|
|
return 0
|
|
|
|
log.info(f"Fetching subscriptions using cookies at {COOKIES_PATH}")
|
|
|
|
cmd = [
|
|
"yt-dlp",
|
|
"--flat-playlist",
|
|
"--dump-json",
|
|
"--cookies", COOKIES_PATH,
|
|
"--playlist-end", "40",
|
|
"--extractor-args", "youtubetab:skip=authcheck",
|
|
"https://www.youtube.com/feed/subscriptions"
|
|
]
|
|
|
|
try:
|
|
process = await asyncio.create_subprocess_exec(
|
|
*cmd,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE
|
|
)
|
|
stdout, stderr = await process.communicate()
|
|
except Exception as e:
|
|
log.error(f"Failed to run yt-dlp: {e}")
|
|
return 0
|
|
|
|
if process.returncode != 0:
|
|
log.error(f"yt-dlp error output: {stderr.decode('utf-8', errors='ignore')}")
|
|
return 0
|
|
|
|
lines = stdout.decode('utf-8', errors='ignore').strip().split('\n')
|
|
new_count = 0
|
|
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
for line in lines:
|
|
if not line:
|
|
continue
|
|
try:
|
|
video = json.loads(line)
|
|
video_id = video.get("id")
|
|
if not video_id:
|
|
continue
|
|
|
|
title = video.get("title", "Untitled")
|
|
channel_name = video.get("uploader") or video.get("channel") or "Unknown Channel"
|
|
channel_id = video.get("uploader_id") or video.get("channel_id")
|
|
url = f"https://www.youtube.com/watch?v={video_id}"
|
|
thumbnail_url = video.get("thumbnail") or f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
|
|
|
|
raw_date = video.get("upload_date")
|
|
if raw_date and len(raw_date) == 8:
|
|
publish_date = f"{raw_date[:4]}-{raw_date[4:6]}-{raw_date[6:]}"
|
|
else:
|
|
publish_date = None
|
|
|
|
duration = video.get("duration")
|
|
|
|
# Check if video already exists in db
|
|
cursor = await db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,))
|
|
exists = await cursor.fetchone()
|
|
if not exists:
|
|
await db.execute("""
|
|
INSERT INTO videos (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration, status)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending')
|
|
""", (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration))
|
|
new_count += 1
|
|
except Exception as e:
|
|
log.warning(f"Error parsing yt-dlp line: {e}")
|
|
continue
|
|
await db.commit()
|
|
|
|
log.info(f"Ingestion completed. Found {new_count} new videos.")
|
|
return new_count
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
async def main():
|
|
await init_db()
|
|
await fetch_subscriptions()
|
|
asyncio.run(main())
|