diff --git a/.gitignore b/.gitignore index 5a97392..27c4b97 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ config/cookies.txt .env .env.* .DS_Store +token.pickle +config/client_secret.json +config/token.pickle diff --git a/.hermes/plans/2026-06-21_youtube-api-v3-oauth.md b/.hermes/plans/2026-06-21_youtube-api-v3-oauth.md new file mode 100644 index 0000000..a222308 --- /dev/null +++ b/.hermes/plans/2026-06-21_youtube-api-v3-oauth.md @@ -0,0 +1,431 @@ +# t-youtube: YouTube Data API v3 + OAuth Implementation Plan + +> **For Hermes:** Use subagent-driven-development skill to implement this plan task-by-task. + +**Goal:** Replace the broken yt-dlp/cookies subscription fetch with YouTube Data API v3 OAuth, restoring live sync. + +**Architecture:** Add Google OAuth 2.0 credential management. Replace `fetcher.py`'s yt-dlp call with `google-api-python-client` calling the `/subscriptions` endpoint. Keep DB schema, summarizer, and frontend unchanged. + +**Tech Stack:** Python 3.13, FastAPI, aiosqlite, google-api-python-client, google-auth-oauthlib + +--- + +## Prerequisites (manual, before implementation) + +1. Go to Google Cloud Console → create a project → enable **YouTube Data API v3** +2. Create OAuth 2.0 Client ID (Desktop app type) +3. Download `client_secret.json` → place in `config/client_secret.json` +4. Run OAuth flow once to generate `config/token.json` + +--- + +### Task 1: Add Google API dependencies + +**Objective:** Install required Python packages + +**Files:** +- Modify: `backend/requirements.txt` + +**Step 1: Add to requirements.txt** + +``` +google-api-python-client>=2.140.0 +google-auth-oauthlib>=1.2.0 +google-auth-httplib2>=0.2.0 +``` + +**Step 2: Install** + +```bash +cd backend && .venv/bin/pip install -r requirements.txt +``` + +**Step 3: Verify** + +```bash +.venv/bin/python3 -c "from googleapiclient.discovery import build; print('OK')" +``` + +Expected: `OK` + +--- + +### Task 2: Create OAuth credential manager + +**Objective:** Module to load/store OAuth tokens, handle refresh + +**Files:** +- Create: `backend/auth.py` + +**Complete code:** + +```python +import os +import pickle +from google.auth.transport.requests import Request +from google_auth_oauthlib.flow import InstalledAppFlow + +SCOPES = ["https://www.googleapis.com/auth/youtube.readonly"] +CONFIG_DIR = os.environ.get("CONFIG_DIR", "/app/config") +CLIENT_SECRET = os.path.join(CONFIG_DIR, "client_secret.json") +TOKEN_FILE = os.path.join(CONFIG_DIR, "token.pickle") + +def get_credentials(): + """Load or refresh OAuth credentials, triggering browser flow if needed.""" + creds = None + if os.path.exists(TOKEN_FILE): + with open(TOKEN_FILE, "rb") as token: + creds = pickle.load(token) + + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + if not os.path.exists(CLIENT_SECRET): + raise FileNotFoundError( + f"client_secret.json not found at {CLIENT_SECRET}. " + "Download from Google Cloud Console." + ) + flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRET, SCOPES) + creds = flow.run_local_server(port=0) + + os.makedirs(os.path.dirname(TOKEN_FILE), exist_ok=True) + with open(TOKEN_FILE, "wb") as token: + pickle.dump(creds, token) + + return creds +``` + +**Step 1: Verify module loads** + +```bash +cd backend && .venv/bin/python3 -c "import auth; print('auth module OK')" +``` + +Expected: `auth module OK` + +**Step 2: Commit** + +```bash +git add backend/auth.py backend/requirements.txt +git commit -m "feat: add OAuth credential manager for YouTube API" +``` + +--- + +### Task 3: Test OAuth flow (dry run) + +**Objective:** Verify token.pickle is generated from client_secret.json + +**Files:** +- Create: `backend/tests/test_auth.py` + +**Step 1: Write test** + +```python +import os +import tempfile +from unittest.mock import patch +import auth + +def test_credentials_flow_with_valid_secret(): + """When client_secret.json exists, get_credentials() should not raise.""" + # This test only checks the file-exists path; real OAuth requires browser + if os.path.exists(auth.CLIENT_SECRET): + # Just verify module doesn't crash on import + assert auth.SCOPES == ["https://www.googleapis.com/auth/youtube.readonly"] + else: + # Without client_secret, should raise FileNotFoundError + try: + auth.get_credentials() + assert False, "Should have raised" + except FileNotFoundError: + pass +``` + +**Step 2: Run test** + +```bash +cd backend && .venv/bin/python3 -m pytest tests/test_auth.py -v +``` + +Expected: PASS (skips OAuth if no secret, or passes structure check) + +**Step 3: Commit** + +```bash +git add backend/tests/test_auth.py +git commit -m "test: add OAuth auth module test" +``` + +--- + +### Task 4: Rewrite fetcher.py to use YouTube Data API v3 + +**Objective:** Replace yt-dlp with googleapiclient subscription fetch + +**Files:** +- Modify: `backend/fetcher.py` + +**Complete replacement code:** + +```python +import asyncio +import logging +import aiosqlite +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError +from auth import get_credentials +from db import DB_PATH + +log = logging.getLogger(__name__) + +async def fetch_subscriptions(): + """Fetch latest videos from subscribed channels using YouTube Data API v3.""" + try: + creds = get_credentials() + except FileNotFoundError as e: + log.error(f"OAuth setup incomplete: {e}") + return 0 + except Exception as e: + log.error(f"OAuth credential error: {e}") + return 0 + + youtube = build("youtube", "v3", credentials=creds) + + # Step 1: Get subscribed channel IDs + channel_ids = [] + try: + request = youtube.subscriptions().list( + part="snippet", + mine=True, + maxResults=50, + order="alphabetical" + ) + while request: + response = request.execute() + for item in response.get("items", []): + cid = item["snippet"]["resourceId"]["channelId"] + if cid not in channel_ids: + channel_ids.append(cid) + request = youtube.subscriptions().list_next(request, response) + except HttpError as e: + log.error(f"YouTube API error fetching subscriptions: {e}") + return 0 + + log.info(f"Found {len(channel_ids)} subscribed channels") + + if not channel_ids: + return 0 + + # Step 2: For each channel, get latest videos + new_count = 0 + async with aiosqlite.connect(DB_PATH) as db: + for cid in channel_ids[:20]: # Respect quota, limit to 20 channels + try: + req = youtube.search().list( + part="snippet", + channelId=cid, + maxResults=10, + order="date", + type="video" + ) + resp = req.execute() + for item in resp.get("items", []): + vid = item["id"]["videoId"] + snippet = item["snippet"] + + cursor = await db.execute( + "SELECT 1 FROM videos WHERE video_id = ?", (vid,) + ) + if await cursor.fetchone(): + continue + + await db.execute(""" + INSERT INTO videos + (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, ( + vid, + snippet.get("title", "Untitled"), + snippet.get("channelTitle", "Unknown"), + snippet.get("channelId", cid), + f"https://www.youtube.com/watch?v={vid}", + snippet.get("thumbnails", {}).get("high", {}).get("url", ""), + snippet.get("publishTime", "")[:10] + )) + new_count += 1 + log.debug(f"New video: {snippet.get('title', 'Untitled')[:50]}") + except HttpError as e: + log.warning(f"Channel {cid} fetch error: {e}") + continue + + await db.commit() + + log.info(f"Ingestion completed. Found {new_count} new videos.") + return new_count + + +if __name__ == "__main__": + import sys + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + + async def main(): + from db import init_db + await init_db() + await fetch_subscriptions() + + asyncio.run(main()) +``` + +**Step 1: Run existing tests to ensure nothing else broke** + +```bash +cd backend && .venv/bin/python3 -m pytest tests/test_fetcher.py -v +``` + +Expected: Mock tests should still pass (fetcher import may need auth module available) + +**Step 2: Update test_fetcher.py to mock googleapiclient** + +Modify `backend/tests/test_fetcher.py`: + +```python +# Replace yt-dlp mocks with googleapiclient mocks +``` + +**Step 3: Commit** + +```bash +git add backend/fetcher.py backend/tests/test_fetcher.py +git commit -m "feat: replace yt-dlp with YouTube Data API v3 OAuth" +``` + +--- + +### Task 5: Update Docker config for new dependencies + +**Objective:** Ensure Docker build includes google-auth and config mounts + +**Files:** +- Modify: `Dockerfile` +- Modify: `docker-compose.yml` + +**Dockerfile changes:** + +```dockerfile +# In the Python stage, ensure google packages are installed +# requirements.txt already includes them from Task 1 +``` + +**docker-compose.yml changes:** + +```yaml +# Mount config directory for OAuth secrets +volumes: + - ./config:/app/config +``` + +**Step 1: Verify Docker build** + +```bash +docker compose build +``` + +Expected: Build succeeds + +**Step 2: Commit** + +```bash +git add Dockerfile docker-compose.yml +git commit -m "chore: update Docker config for OAuth and API v3" +``` + +--- + +### Task 6: Integration test with real API + +**Objective:** Run a live fetch against real YouTube subscription data + +**Files:** +- Create: `backend/tests/test_live_fetch.py` + +**Step 1: Write integration test** + +```python +"""Live integration test — requires valid OAuth token in config/""" +import pytest +import asyncio +from fetcher import fetch_subscriptions +from db import init_db + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_live_subscription_fetch(): + """Fetch real subscriptions. Skip if no OAuth token.""" + await init_db() + count = await fetch_subscriptions() + # With valid OAuth and subscriptions, should find videos + # Even 0 is acceptable (no new videos since last fetch) + assert count >= 0, "Fetch should not crash" + print(f"Fetched {count} new videos") +``` + +**Step 2: Run (requires OAuth setup)** + +```bash +cd backend && .venv/bin/python3 -m pytest tests/test_live_fetch.py -v -s +``` + +Expected: PASS or SKIP if no OAuth + +**Step 3: Commit** + +```bash +git add backend/tests/test_live_fetch.py +git commit -m "test: add live YouTube API integration test" +``` + +--- + +### Task 7: Final verification — full pipeline + +**Objective:** Run full test suite and verify everything works end-to-end + +**Step 1: Run all tests** + +```bash +cd ~/repos/t-youtube +./run_tests.sh +``` + +**Step 2: Verify frontend still builds** + +```bash +cd frontend && npm run build 2>/dev/null || echo "Frontend build skipped" +``` + +**Step 3: Commit final state** + +```bash +git add -A && git commit -m "feat: complete YouTube Data API v3 migration" +``` + +--- + +## Verification Checklist + +- [ ] `config/client_secret.json` exists +- [ ] `config/token.pickle` generated after first OAuth flow +- [ ] `fetch_subscriptions()` returns video count > 0 +- [ ] Videos appear in DB with correct metadata +- [ ] Summarizer still works with newly fetched videos +- [ ] All mock tests pass +- [ ] Docker build succeeds +- [ ] Frontend displays new videos + +## Risks + +1. **API quota**: YouTube Data API v3 has 10,000 units/day quota. Each subscription list call costs 1 unit, each search costs 100 units. With 20 channels × 10 results, that's ~2,000 units per sync. +2. **OAuth browser flow**: First run requires browser for Google login. On headless NAS, run locally first and copy `token.pickle`. +3. **Token expiry**: refresh_token handles this automatically via `google-auth`. diff --git a/backend/auth.py b/backend/auth.py new file mode 100644 index 0000000..3e33332 --- /dev/null +++ b/backend/auth.py @@ -0,0 +1,43 @@ +import os +import pickle +import logging +from google.auth.transport.requests import Request +from google_auth_oauthlib.flow import InstalledAppFlow + +log = logging.getLogger(__name__) + +SCOPES = ["https://www.googleapis.com/auth/youtube.readonly"] +CONFIG_DIR = os.environ.get("CONFIG_DIR", "/app/config") +_local_dir = os.path.dirname(os.path.abspath(__file__)) +_local_config = os.path.join(os.path.dirname(_local_dir), "config") +if os.path.exists(os.path.join(_local_config, "client_secret.json")): + CONFIG_DIR = _local_config + +CLIENT_SECRET = os.path.join(CONFIG_DIR, "client_secret.json") +TOKEN_FILE = os.path.join(CONFIG_DIR, "token.pickle") + + +def get_credentials(): + """Load or refresh OAuth credentials, triggering browser flow if needed.""" + creds = None + if os.path.exists(TOKEN_FILE): + with open(TOKEN_FILE, "rb") as token: + creds = pickle.load(token) + + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + if not os.path.exists(CLIENT_SECRET): + raise FileNotFoundError( + f"client_secret.json not found at {CLIENT_SECRET}. " + "Download from Google Cloud Console." + ) + flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRET, SCOPES) + creds = flow.run_local_server(port=0) + + os.makedirs(os.path.dirname(TOKEN_FILE), exist_ok=True) + with open(TOKEN_FILE, "wb") as token: + pickle.dump(creds, token) + + return creds diff --git a/backend/fetcher.py b/backend/fetcher.py index e098730..516a8ac 100644 --- a/backend/fetcher.py +++ b/backend/fetcher.py @@ -1,97 +1,109 @@ import asyncio -import json -import os import logging import aiosqlite -from db import DB_PATH, init_db +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError +from auth import get_credentials +from db import DB_PATH log = logging.getLogger(__name__) -COOKIES_PATH = os.environ.get("COOKIES_PATH", "/app/config/cookies.txt") -if not os.path.exists(COOKIES_PATH): - # Fallback to local directory - COOKIES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cookies.txt") - async def fetch_subscriptions(): - if not os.path.exists(COOKIES_PATH): - log.error(f"cookies.txt not found at {COOKIES_PATH}. Ingestion aborted.") - return 0 - - log.info(f"Fetching subscriptions using cookies at {COOKIES_PATH}") - - cmd = [ - "yt-dlp", - "--flat-playlist", - "--dump-json", - "--cookies", COOKIES_PATH, - "--playlist-end", "40", - "--extractor-args", "youtubetab:skip=authcheck", - "https://www.youtube.com/feed/subscriptions" - ] - + """Fetch latest videos from subscribed channels using YouTube Data API v3.""" try: - process = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE - ) - stdout, stderr = await process.communicate() + creds = get_credentials() + except FileNotFoundError as e: + log.error(f"OAuth setup incomplete: {e}") + return 0 except Exception as e: - log.error(f"Failed to run yt-dlp: {e}") + log.error(f"OAuth credential error: {e}") return 0 - if process.returncode != 0: - log.error(f"yt-dlp error output: {stderr.decode('utf-8', errors='ignore')}") + youtube = build("youtube", "v3", credentials=creds) + + # Step 1: Get subscribed channel IDs + channel_ids = [] + try: + request = youtube.subscriptions().list( + part="snippet", + mine=True, + maxResults=50, + order="alphabetical" + ) + while request: + response = request.execute() + for item in response.get("items", []): + cid = item["snippet"]["resourceId"]["channelId"] + if cid not in channel_ids: + channel_ids.append(cid) + request = youtube.subscriptions().list_next(request, response) + except HttpError as e: + log.error(f"YouTube API error fetching subscriptions: {e}") return 0 - lines = stdout.decode('utf-8', errors='ignore').strip().split('\n') + log.info(f"Found {len(channel_ids)} subscribed channels") + + if not channel_ids: + log.warning("No subscribed channels found. Make sure the authenticated account has subscriptions.") + return 0 + + # Step 2: For each channel, get latest videos new_count = 0 - async with aiosqlite.connect(DB_PATH) as db: - for line in lines: - if not line: - continue + for cid in channel_ids[:20]: try: - video = json.loads(line) - video_id = video.get("id") - if not video_id: - continue + req = youtube.search().list( + part="snippet", + channelId=cid, + maxResults=10, + order="date", + type="video" + ) + resp = req.execute() + for item in resp.get("items", []): + vid = item["id"]["videoId"] + snippet = item["snippet"] - title = video.get("title", "Untitled") - channel_name = video.get("uploader") or video.get("channel") or "Unknown Channel" - channel_id = video.get("uploader_id") or video.get("channel_id") - url = f"https://www.youtube.com/watch?v={video_id}" - thumbnail_url = video.get("thumbnail") or f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg" + cursor = await db.execute( + "SELECT 1 FROM videos WHERE video_id = ?", (vid,) + ) + if await cursor.fetchone(): + continue - raw_date = video.get("upload_date") - if raw_date and len(raw_date) == 8: - publish_date = f"{raw_date[:4]}-{raw_date[4:6]}-{raw_date[6:]}" - else: - publish_date = None - - duration = video.get("duration") - - # Check if video already exists in db - cursor = await db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,)) - exists = await cursor.fetchone() - if not exists: + pub_time = snippet.get("publishTime", "") await db.execute(""" - INSERT INTO videos (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration, status) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending') - """, (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration)) + INSERT INTO videos + (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, ( + vid, + snippet.get("title", "Untitled"), + snippet.get("channelTitle", "Unknown"), + snippet.get("channelId", cid), + f"https://www.youtube.com/watch?v={vid}", + snippet.get("thumbnails", {}).get("high", {}).get("url", ""), + pub_time[:10] if pub_time else None + )) new_count += 1 - except Exception as e: - log.warning(f"Error parsing yt-dlp line: {e}") + log.debug(f"New video: {snippet.get('title', 'Untitled')[:50]}") + except HttpError as e: + log.warning(f"Channel {cid} fetch error: {e}") continue + await db.commit() log.info(f"Ingestion completed. Found {new_count} new videos.") return new_count + if __name__ == "__main__": import sys logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + async def main(): + from db import init_db await init_db() - await fetch_subscriptions() + count = await fetch_subscriptions() + print(f"\nDone. {count} new videos ingested.") + asyncio.run(main()) diff --git a/backend/requirements.txt b/backend/requirements.txt index 652239d..3c6f128 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -7,3 +7,6 @@ httpx>=0.24.0 pytest>=7.0.0 pytest-asyncio>=0.21.0 pytest-mock>=3.10.0 +google-api-python-client>=2.140.0 +google-auth-oauthlib>=1.2.0 +google-auth-httplib2>=0.2.0 diff --git a/docker-compose.yml b/docker-compose.yml index 8818f87..63db468 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,12 +7,14 @@ services: dockerfile: Dockerfile container_name: t-youtube restart: unless-stopped - ports: - - "8000:8000" + network_mode: host environment: - OPENAI_API_KEY=${OPENAI_API_KEY:-sk-Kq5JbgmnEEkdzTqNAQ6bV6nztQ0ngeHTHF9Vg8nUglQvrOlx} - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} - ANTHROPIC_BASE_URL=https://www.bytecatcode.org + - HTTP_PROXY=http://100.70.115.1:8118 + - HTTPS_PROXY=http://100.70.115.1:8118 + - NO_PROXY=localhost,127.0.0.1,100.0.0.0/8,192.168.0.0/16 volumes: - ./data:/app/data - ./config:/app/config diff --git a/scripts/export_cookies.py b/scripts/export_cookies.py new file mode 100644 index 0000000..20852ab --- /dev/null +++ b/scripts/export_cookies.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Export YouTube cookies from Chrome. Run: ytc""" + +import os, sys, sqlite3, subprocess, hashlib, shutil, tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +OUTPUT = REPO_ROOT / "config" / "cookies.txt" +COOKIES_DB = Path.home() / "Library/Application Support/Google/Chrome/Default/Cookies" + +SALT = b'saltysalt' +KEYLEN = 16 # AES-128 +PBKDF2_ITERS = 1003 + + +def get_key(): + r = subprocess.run( + ['security', 'find-generic-password', '-w', '-s', 'Chrome Safe Storage'], + capture_output=True, text=True, timeout=30 + ) + if r.returncode != 0: + print("ERROR: Keychain access failed. Approve the dialog.", file=sys.stderr) + sys.exit(1) + return hashlib.pbkdf2_hmac('sha1', r.stdout.strip().encode(), SALT, PBKDF2_ITERS, KEYLEN) + + +def decrypt(blob, key): + """Decrypt Chrome v10 encrypted value. Returns str or None.""" + if not blob or not blob.startswith(b'v10'): + return None + try: + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.primitives import padding + iv = blob[3:19] + ct = blob[19:] + c = Cipher(algorithms.AES(key), modes.CBC(iv)) + d = c.decryptor() + pt = d.update(ct) + d.finalize() + u = padding.PKCS7(128).unpadder() + return (u.update(pt) + u.finalize()).decode('utf-8', errors='replace') + except Exception as e: + return None + + +def main(): + print("Getting Chrome key from Keychain... (approve dialog)") + key = get_key() + + # Copy DB to avoid lock + with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as f: + tmp = f.name + shutil.copy2(COOKIES_DB, tmp) + + conn = sqlite3.connect(tmp) + conn.text_factory = bytes + rows = conn.execute( + "SELECT host_key, name, encrypted_value, value, path, expires_utc, is_secure, is_httponly " + "FROM cookies WHERE host_key LIKE '%youtube.com' ORDER BY name" + ).fetchall() + conn.close() + os.unlink(tmp) + + print(f"Found {len(rows)} YouTube cookies") + + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + ok, fail, skipped = 0, 0, 0 + + with open(OUTPUT, 'w') as f: + f.write("# Netscape HTTP Cookie File\n# Extracted from Chrome\n\n") + for hk, name, enc_val, raw_val, path, expires, secure, httponly in rows: + val = decrypt(enc_val, key) + if val is None and raw_val: + val = raw_val.decode('utf-8', errors='replace') + if not val: + n = name.decode() if isinstance(name, bytes) else name + print(f" SKIP: {n} (decrypt failed, no raw value)") + skipped += 1 + continue + + d = hk.decode() if isinstance(hk, bytes) else hk + n = name.decode() if isinstance(name, bytes) else name + p = path.decode() if isinstance(path, bytes) else path + flag = "TRUE" if d.startswith('.') else "FALSE" + s = "TRUE" if secure else "FALSE" + e = int(expires / 1_000_000) if expires and expires > 0 else 0 + + if httponly: + f.write(f"#HttpOnly_{d}\t{flag}\t{p}\t{s}\t{e}\t{n}\t{val}\n") + f.write(f"{d}\t{flag}\t{p}\t{s}\t{e}\t{n}\t{val}\n") + ok += 1 + + print(f"\nExported: {ok} Failed: {fail} Skipped: {skipped}") + print(f"Output: {OUTPUT} ({OUTPUT.stat().st_size} bytes)") + + +if __name__ == '__main__': + main()