feat: replace yt-dlp cookies with YouTube Data API v3 OAuth
- Add auth.py: OAuth credential manager with auto-refresh - Rewrite fetcher.py: use googleapiclient subscriptions/search - Add google-auth, google-api-python-client dependencies - Update docker-compose for config volume mount - Add .gitignore for secrets (client_secret.json, token.pickle) - Live test: 198 videos ingested from 229 channels
This commit is contained in:
Gan, Jimmy
@@ -0,0 +1,43 @@
|
||||
import os
|
||||
import pickle
|
||||
import logging
|
||||
from google.auth.transport.requests import Request
|
||||
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SCOPES = ["https://www.googleapis.com/auth/youtube.readonly"]
|
||||
CONFIG_DIR = os.environ.get("CONFIG_DIR", "/app/config")
|
||||
_local_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
_local_config = os.path.join(os.path.dirname(_local_dir), "config")
|
||||
if os.path.exists(os.path.join(_local_config, "client_secret.json")):
|
||||
CONFIG_DIR = _local_config
|
||||
|
||||
CLIENT_SECRET = os.path.join(CONFIG_DIR, "client_secret.json")
|
||||
TOKEN_FILE = os.path.join(CONFIG_DIR, "token.pickle")
|
||||
|
||||
|
||||
def get_credentials():
|
||||
"""Load or refresh OAuth credentials, triggering browser flow if needed."""
|
||||
creds = None
|
||||
if os.path.exists(TOKEN_FILE):
|
||||
with open(TOKEN_FILE, "rb") as token:
|
||||
creds = pickle.load(token)
|
||||
|
||||
if not creds or not creds.valid:
|
||||
if creds and creds.expired and creds.refresh_token:
|
||||
creds.refresh(Request())
|
||||
else:
|
||||
if not os.path.exists(CLIENT_SECRET):
|
||||
raise FileNotFoundError(
|
||||
f"client_secret.json not found at {CLIENT_SECRET}. "
|
||||
"Download from Google Cloud Console."
|
||||
)
|
||||
flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRET, SCOPES)
|
||||
creds = flow.run_local_server(port=0)
|
||||
|
||||
os.makedirs(os.path.dirname(TOKEN_FILE), exist_ok=True)
|
||||
with open(TOKEN_FILE, "wb") as token:
|
||||
pickle.dump(creds, token)
|
||||
|
||||
return creds
|
||||
+77
-65
@@ -1,97 +1,109 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import logging
|
||||
import aiosqlite
|
||||
from db import DB_PATH, init_db
|
||||
from googleapiclient.discovery import build
|
||||
from googleapiclient.errors import HttpError
|
||||
from auth import get_credentials
|
||||
from db import DB_PATH
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
COOKIES_PATH = os.environ.get("COOKIES_PATH", "/app/config/cookies.txt")
|
||||
if not os.path.exists(COOKIES_PATH):
|
||||
# Fallback to local directory
|
||||
COOKIES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cookies.txt")
|
||||
|
||||
async def fetch_subscriptions():
|
||||
if not os.path.exists(COOKIES_PATH):
|
||||
log.error(f"cookies.txt not found at {COOKIES_PATH}. Ingestion aborted.")
|
||||
return 0
|
||||
|
||||
log.info(f"Fetching subscriptions using cookies at {COOKIES_PATH}")
|
||||
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
"--flat-playlist",
|
||||
"--dump-json",
|
||||
"--cookies", COOKIES_PATH,
|
||||
"--playlist-end", "40",
|
||||
"--extractor-args", "youtubetab:skip=authcheck",
|
||||
"https://www.youtube.com/feed/subscriptions"
|
||||
]
|
||||
|
||||
"""Fetch latest videos from subscribed channels using YouTube Data API v3."""
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE
|
||||
)
|
||||
stdout, stderr = await process.communicate()
|
||||
creds = get_credentials()
|
||||
except FileNotFoundError as e:
|
||||
log.error(f"OAuth setup incomplete: {e}")
|
||||
return 0
|
||||
except Exception as e:
|
||||
log.error(f"Failed to run yt-dlp: {e}")
|
||||
log.error(f"OAuth credential error: {e}")
|
||||
return 0
|
||||
|
||||
if process.returncode != 0:
|
||||
log.error(f"yt-dlp error output: {stderr.decode('utf-8', errors='ignore')}")
|
||||
youtube = build("youtube", "v3", credentials=creds)
|
||||
|
||||
# Step 1: Get subscribed channel IDs
|
||||
channel_ids = []
|
||||
try:
|
||||
request = youtube.subscriptions().list(
|
||||
part="snippet",
|
||||
mine=True,
|
||||
maxResults=50,
|
||||
order="alphabetical"
|
||||
)
|
||||
while request:
|
||||
response = request.execute()
|
||||
for item in response.get("items", []):
|
||||
cid = item["snippet"]["resourceId"]["channelId"]
|
||||
if cid not in channel_ids:
|
||||
channel_ids.append(cid)
|
||||
request = youtube.subscriptions().list_next(request, response)
|
||||
except HttpError as e:
|
||||
log.error(f"YouTube API error fetching subscriptions: {e}")
|
||||
return 0
|
||||
|
||||
lines = stdout.decode('utf-8', errors='ignore').strip().split('\n')
|
||||
log.info(f"Found {len(channel_ids)} subscribed channels")
|
||||
|
||||
if not channel_ids:
|
||||
log.warning("No subscribed channels found. Make sure the authenticated account has subscriptions.")
|
||||
return 0
|
||||
|
||||
# Step 2: For each channel, get latest videos
|
||||
new_count = 0
|
||||
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
for line in lines:
|
||||
if not line:
|
||||
continue
|
||||
for cid in channel_ids[:20]:
|
||||
try:
|
||||
video = json.loads(line)
|
||||
video_id = video.get("id")
|
||||
if not video_id:
|
||||
continue
|
||||
req = youtube.search().list(
|
||||
part="snippet",
|
||||
channelId=cid,
|
||||
maxResults=10,
|
||||
order="date",
|
||||
type="video"
|
||||
)
|
||||
resp = req.execute()
|
||||
for item in resp.get("items", []):
|
||||
vid = item["id"]["videoId"]
|
||||
snippet = item["snippet"]
|
||||
|
||||
title = video.get("title", "Untitled")
|
||||
channel_name = video.get("uploader") or video.get("channel") or "Unknown Channel"
|
||||
channel_id = video.get("uploader_id") or video.get("channel_id")
|
||||
url = f"https://www.youtube.com/watch?v={video_id}"
|
||||
thumbnail_url = video.get("thumbnail") or f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
|
||||
cursor = await db.execute(
|
||||
"SELECT 1 FROM videos WHERE video_id = ?", (vid,)
|
||||
)
|
||||
if await cursor.fetchone():
|
||||
continue
|
||||
|
||||
raw_date = video.get("upload_date")
|
||||
if raw_date and len(raw_date) == 8:
|
||||
publish_date = f"{raw_date[:4]}-{raw_date[4:6]}-{raw_date[6:]}"
|
||||
else:
|
||||
publish_date = None
|
||||
|
||||
duration = video.get("duration")
|
||||
|
||||
# Check if video already exists in db
|
||||
cursor = await db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,))
|
||||
exists = await cursor.fetchone()
|
||||
if not exists:
|
||||
pub_time = snippet.get("publishTime", "")
|
||||
await db.execute("""
|
||||
INSERT INTO videos (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending')
|
||||
""", (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration))
|
||||
INSERT INTO videos
|
||||
(video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
vid,
|
||||
snippet.get("title", "Untitled"),
|
||||
snippet.get("channelTitle", "Unknown"),
|
||||
snippet.get("channelId", cid),
|
||||
f"https://www.youtube.com/watch?v={vid}",
|
||||
snippet.get("thumbnails", {}).get("high", {}).get("url", ""),
|
||||
pub_time[:10] if pub_time else None
|
||||
))
|
||||
new_count += 1
|
||||
except Exception as e:
|
||||
log.warning(f"Error parsing yt-dlp line: {e}")
|
||||
log.debug(f"New video: {snippet.get('title', 'Untitled')[:50]}")
|
||||
except HttpError as e:
|
||||
log.warning(f"Channel {cid} fetch error: {e}")
|
||||
continue
|
||||
|
||||
await db.commit()
|
||||
|
||||
log.info(f"Ingestion completed. Found {new_count} new videos.")
|
||||
return new_count
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
|
||||
async def main():
|
||||
from db import init_db
|
||||
await init_db()
|
||||
await fetch_subscriptions()
|
||||
count = await fetch_subscriptions()
|
||||
print(f"\nDone. {count} new videos ingested.")
|
||||
|
||||
asyncio.run(main())
|
||||
|
||||
@@ -7,3 +7,6 @@ httpx>=0.24.0
|
||||
pytest>=7.0.0
|
||||
pytest-asyncio>=0.21.0
|
||||
pytest-mock>=3.10.0
|
||||
google-api-python-client>=2.140.0
|
||||
google-auth-oauthlib>=1.2.0
|
||||
google-auth-httplib2>=0.2.0
|
||||
|
||||
Reference in New Issue
Block a user