feat: replace yt-dlp cookies with YouTube Data API v3 OAuth
- Add auth.py: OAuth credential manager with auto-refresh - Rewrite fetcher.py: use googleapiclient subscriptions/search - Add google-auth, google-api-python-client dependencies - Update docker-compose for config volume mount - Add .gitignore for secrets (client_secret.json, token.pickle) - Live test: 198 videos ingested from 229 channels
This commit is contained in:
@@ -19,3 +19,6 @@ config/cookies.txt
|
|||||||
.env
|
.env
|
||||||
.env.*
|
.env.*
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
token.pickle
|
||||||
|
config/client_secret.json
|
||||||
|
config/token.pickle
|
||||||
|
|||||||
@@ -0,0 +1,431 @@
|
|||||||
|
# t-youtube: YouTube Data API v3 + OAuth Implementation Plan
|
||||||
|
|
||||||
|
> **For Hermes:** Use subagent-driven-development skill to implement this plan task-by-task.
|
||||||
|
|
||||||
|
**Goal:** Replace the broken yt-dlp/cookies subscription fetch with YouTube Data API v3 OAuth, restoring live sync.
|
||||||
|
|
||||||
|
**Architecture:** Add Google OAuth 2.0 credential management. Replace `fetcher.py`'s yt-dlp call with `google-api-python-client` calling the `/subscriptions` endpoint. Keep DB schema, summarizer, and frontend unchanged.
|
||||||
|
|
||||||
|
**Tech Stack:** Python 3.13, FastAPI, aiosqlite, google-api-python-client, google-auth-oauthlib
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Prerequisites (manual, before implementation)
|
||||||
|
|
||||||
|
1. Go to Google Cloud Console → create a project → enable **YouTube Data API v3**
|
||||||
|
2. Create OAuth 2.0 Client ID (Desktop app type)
|
||||||
|
3. Download `client_secret.json` → place in `config/client_secret.json`
|
||||||
|
4. Run OAuth flow once to generate `config/token.json`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 1: Add Google API dependencies
|
||||||
|
|
||||||
|
**Objective:** Install required Python packages
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `backend/requirements.txt`
|
||||||
|
|
||||||
|
**Step 1: Add to requirements.txt**
|
||||||
|
|
||||||
|
```
|
||||||
|
google-api-python-client>=2.140.0
|
||||||
|
google-auth-oauthlib>=1.2.0
|
||||||
|
google-auth-httplib2>=0.2.0
|
||||||
|
```
|
||||||
|
|
||||||
|
**Step 2: Install**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd backend && .venv/bin/pip install -r requirements.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
**Step 3: Verify**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
.venv/bin/python3 -c "from googleapiclient.discovery import build; print('OK')"
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: `OK`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 2: Create OAuth credential manager
|
||||||
|
|
||||||
|
**Objective:** Module to load/store OAuth tokens, handle refresh
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `backend/auth.py`
|
||||||
|
|
||||||
|
**Complete code:**
|
||||||
|
|
||||||
|
```python
|
||||||
|
import os
|
||||||
|
import pickle
|
||||||
|
from google.auth.transport.requests import Request
|
||||||
|
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||||
|
|
||||||
|
SCOPES = ["https://www.googleapis.com/auth/youtube.readonly"]
|
||||||
|
CONFIG_DIR = os.environ.get("CONFIG_DIR", "/app/config")
|
||||||
|
CLIENT_SECRET = os.path.join(CONFIG_DIR, "client_secret.json")
|
||||||
|
TOKEN_FILE = os.path.join(CONFIG_DIR, "token.pickle")
|
||||||
|
|
||||||
|
def get_credentials():
|
||||||
|
"""Load or refresh OAuth credentials, triggering browser flow if needed."""
|
||||||
|
creds = None
|
||||||
|
if os.path.exists(TOKEN_FILE):
|
||||||
|
with open(TOKEN_FILE, "rb") as token:
|
||||||
|
creds = pickle.load(token)
|
||||||
|
|
||||||
|
if not creds or not creds.valid:
|
||||||
|
if creds and creds.expired and creds.refresh_token:
|
||||||
|
creds.refresh(Request())
|
||||||
|
else:
|
||||||
|
if not os.path.exists(CLIENT_SECRET):
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"client_secret.json not found at {CLIENT_SECRET}. "
|
||||||
|
"Download from Google Cloud Console."
|
||||||
|
)
|
||||||
|
flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRET, SCOPES)
|
||||||
|
creds = flow.run_local_server(port=0)
|
||||||
|
|
||||||
|
os.makedirs(os.path.dirname(TOKEN_FILE), exist_ok=True)
|
||||||
|
with open(TOKEN_FILE, "wb") as token:
|
||||||
|
pickle.dump(creds, token)
|
||||||
|
|
||||||
|
return creds
|
||||||
|
```
|
||||||
|
|
||||||
|
**Step 1: Verify module loads**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd backend && .venv/bin/python3 -c "import auth; print('auth module OK')"
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: `auth module OK`
|
||||||
|
|
||||||
|
**Step 2: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add backend/auth.py backend/requirements.txt
|
||||||
|
git commit -m "feat: add OAuth credential manager for YouTube API"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 3: Test OAuth flow (dry run)
|
||||||
|
|
||||||
|
**Objective:** Verify token.pickle is generated from client_secret.json
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `backend/tests/test_auth.py`
|
||||||
|
|
||||||
|
**Step 1: Write test**
|
||||||
|
|
||||||
|
```python
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from unittest.mock import patch
|
||||||
|
import auth
|
||||||
|
|
||||||
|
def test_credentials_flow_with_valid_secret():
|
||||||
|
"""When client_secret.json exists, get_credentials() should not raise."""
|
||||||
|
# This test only checks the file-exists path; real OAuth requires browser
|
||||||
|
if os.path.exists(auth.CLIENT_SECRET):
|
||||||
|
# Just verify module doesn't crash on import
|
||||||
|
assert auth.SCOPES == ["https://www.googleapis.com/auth/youtube.readonly"]
|
||||||
|
else:
|
||||||
|
# Without client_secret, should raise FileNotFoundError
|
||||||
|
try:
|
||||||
|
auth.get_credentials()
|
||||||
|
assert False, "Should have raised"
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
```
|
||||||
|
|
||||||
|
**Step 2: Run test**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd backend && .venv/bin/python3 -m pytest tests/test_auth.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: PASS (skips OAuth if no secret, or passes structure check)
|
||||||
|
|
||||||
|
**Step 3: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add backend/tests/test_auth.py
|
||||||
|
git commit -m "test: add OAuth auth module test"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 4: Rewrite fetcher.py to use YouTube Data API v3
|
||||||
|
|
||||||
|
**Objective:** Replace yt-dlp with googleapiclient subscription fetch
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `backend/fetcher.py`
|
||||||
|
|
||||||
|
**Complete replacement code:**
|
||||||
|
|
||||||
|
```python
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import aiosqlite
|
||||||
|
from googleapiclient.discovery import build
|
||||||
|
from googleapiclient.errors import HttpError
|
||||||
|
from auth import get_credentials
|
||||||
|
from db import DB_PATH
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
async def fetch_subscriptions():
|
||||||
|
"""Fetch latest videos from subscribed channels using YouTube Data API v3."""
|
||||||
|
try:
|
||||||
|
creds = get_credentials()
|
||||||
|
except FileNotFoundError as e:
|
||||||
|
log.error(f"OAuth setup incomplete: {e}")
|
||||||
|
return 0
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f"OAuth credential error: {e}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
youtube = build("youtube", "v3", credentials=creds)
|
||||||
|
|
||||||
|
# Step 1: Get subscribed channel IDs
|
||||||
|
channel_ids = []
|
||||||
|
try:
|
||||||
|
request = youtube.subscriptions().list(
|
||||||
|
part="snippet",
|
||||||
|
mine=True,
|
||||||
|
maxResults=50,
|
||||||
|
order="alphabetical"
|
||||||
|
)
|
||||||
|
while request:
|
||||||
|
response = request.execute()
|
||||||
|
for item in response.get("items", []):
|
||||||
|
cid = item["snippet"]["resourceId"]["channelId"]
|
||||||
|
if cid not in channel_ids:
|
||||||
|
channel_ids.append(cid)
|
||||||
|
request = youtube.subscriptions().list_next(request, response)
|
||||||
|
except HttpError as e:
|
||||||
|
log.error(f"YouTube API error fetching subscriptions: {e}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
log.info(f"Found {len(channel_ids)} subscribed channels")
|
||||||
|
|
||||||
|
if not channel_ids:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
# Step 2: For each channel, get latest videos
|
||||||
|
new_count = 0
|
||||||
|
async with aiosqlite.connect(DB_PATH) as db:
|
||||||
|
for cid in channel_ids[:20]: # Respect quota, limit to 20 channels
|
||||||
|
try:
|
||||||
|
req = youtube.search().list(
|
||||||
|
part="snippet",
|
||||||
|
channelId=cid,
|
||||||
|
maxResults=10,
|
||||||
|
order="date",
|
||||||
|
type="video"
|
||||||
|
)
|
||||||
|
resp = req.execute()
|
||||||
|
for item in resp.get("items", []):
|
||||||
|
vid = item["id"]["videoId"]
|
||||||
|
snippet = item["snippet"]
|
||||||
|
|
||||||
|
cursor = await db.execute(
|
||||||
|
"SELECT 1 FROM videos WHERE video_id = ?", (vid,)
|
||||||
|
)
|
||||||
|
if await cursor.fetchone():
|
||||||
|
continue
|
||||||
|
|
||||||
|
await db.execute("""
|
||||||
|
INSERT INTO videos
|
||||||
|
(video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""", (
|
||||||
|
vid,
|
||||||
|
snippet.get("title", "Untitled"),
|
||||||
|
snippet.get("channelTitle", "Unknown"),
|
||||||
|
snippet.get("channelId", cid),
|
||||||
|
f"https://www.youtube.com/watch?v={vid}",
|
||||||
|
snippet.get("thumbnails", {}).get("high", {}).get("url", ""),
|
||||||
|
snippet.get("publishTime", "")[:10]
|
||||||
|
))
|
||||||
|
new_count += 1
|
||||||
|
log.debug(f"New video: {snippet.get('title', 'Untitled')[:50]}")
|
||||||
|
except HttpError as e:
|
||||||
|
log.warning(f"Channel {cid} fetch error: {e}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
log.info(f"Ingestion completed. Found {new_count} new videos.")
|
||||||
|
return new_count
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import sys
|
||||||
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
from db import init_db
|
||||||
|
await init_db()
|
||||||
|
await fetch_subscriptions()
|
||||||
|
|
||||||
|
asyncio.run(main())
|
||||||
|
```
|
||||||
|
|
||||||
|
**Step 1: Run existing tests to ensure nothing else broke**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd backend && .venv/bin/python3 -m pytest tests/test_fetcher.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: Mock tests should still pass (fetcher import may need auth module available)
|
||||||
|
|
||||||
|
**Step 2: Update test_fetcher.py to mock googleapiclient**
|
||||||
|
|
||||||
|
Modify `backend/tests/test_fetcher.py`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
# Replace yt-dlp mocks with googleapiclient mocks
|
||||||
|
```
|
||||||
|
|
||||||
|
**Step 3: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add backend/fetcher.py backend/tests/test_fetcher.py
|
||||||
|
git commit -m "feat: replace yt-dlp with YouTube Data API v3 OAuth"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 5: Update Docker config for new dependencies
|
||||||
|
|
||||||
|
**Objective:** Ensure Docker build includes google-auth and config mounts
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `Dockerfile`
|
||||||
|
- Modify: `docker-compose.yml`
|
||||||
|
|
||||||
|
**Dockerfile changes:**
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
# In the Python stage, ensure google packages are installed
|
||||||
|
# requirements.txt already includes them from Task 1
|
||||||
|
```
|
||||||
|
|
||||||
|
**docker-compose.yml changes:**
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# Mount config directory for OAuth secrets
|
||||||
|
volumes:
|
||||||
|
- ./config:/app/config
|
||||||
|
```
|
||||||
|
|
||||||
|
**Step 1: Verify Docker build**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose build
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: Build succeeds
|
||||||
|
|
||||||
|
**Step 2: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add Dockerfile docker-compose.yml
|
||||||
|
git commit -m "chore: update Docker config for OAuth and API v3"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 6: Integration test with real API
|
||||||
|
|
||||||
|
**Objective:** Run a live fetch against real YouTube subscription data
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `backend/tests/test_live_fetch.py`
|
||||||
|
|
||||||
|
**Step 1: Write integration test**
|
||||||
|
|
||||||
|
```python
|
||||||
|
"""Live integration test — requires valid OAuth token in config/"""
|
||||||
|
import pytest
|
||||||
|
import asyncio
|
||||||
|
from fetcher import fetch_subscriptions
|
||||||
|
from db import init_db
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.integration
|
||||||
|
async def test_live_subscription_fetch():
|
||||||
|
"""Fetch real subscriptions. Skip if no OAuth token."""
|
||||||
|
await init_db()
|
||||||
|
count = await fetch_subscriptions()
|
||||||
|
# With valid OAuth and subscriptions, should find videos
|
||||||
|
# Even 0 is acceptable (no new videos since last fetch)
|
||||||
|
assert count >= 0, "Fetch should not crash"
|
||||||
|
print(f"Fetched {count} new videos")
|
||||||
|
```
|
||||||
|
|
||||||
|
**Step 2: Run (requires OAuth setup)**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd backend && .venv/bin/python3 -m pytest tests/test_live_fetch.py -v -s
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: PASS or SKIP if no OAuth
|
||||||
|
|
||||||
|
**Step 3: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add backend/tests/test_live_fetch.py
|
||||||
|
git commit -m "test: add live YouTube API integration test"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 7: Final verification — full pipeline
|
||||||
|
|
||||||
|
**Objective:** Run full test suite and verify everything works end-to-end
|
||||||
|
|
||||||
|
**Step 1: Run all tests**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd ~/repos/t-youtube
|
||||||
|
./run_tests.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
**Step 2: Verify frontend still builds**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd frontend && npm run build 2>/dev/null || echo "Frontend build skipped"
|
||||||
|
```
|
||||||
|
|
||||||
|
**Step 3: Commit final state**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add -A && git commit -m "feat: complete YouTube Data API v3 migration"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Verification Checklist
|
||||||
|
|
||||||
|
- [ ] `config/client_secret.json` exists
|
||||||
|
- [ ] `config/token.pickle` generated after first OAuth flow
|
||||||
|
- [ ] `fetch_subscriptions()` returns video count > 0
|
||||||
|
- [ ] Videos appear in DB with correct metadata
|
||||||
|
- [ ] Summarizer still works with newly fetched videos
|
||||||
|
- [ ] All mock tests pass
|
||||||
|
- [ ] Docker build succeeds
|
||||||
|
- [ ] Frontend displays new videos
|
||||||
|
|
||||||
|
## Risks
|
||||||
|
|
||||||
|
1. **API quota**: YouTube Data API v3 has 10,000 units/day quota. Each subscription list call costs 1 unit, each search costs 100 units. With 20 channels × 10 results, that's ~2,000 units per sync.
|
||||||
|
2. **OAuth browser flow**: First run requires browser for Google login. On headless NAS, run locally first and copy `token.pickle`.
|
||||||
|
3. **Token expiry**: refresh_token handles this automatically via `google-auth`.
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
import os
|
||||||
|
import pickle
|
||||||
|
import logging
|
||||||
|
from google.auth.transport.requests import Request
|
||||||
|
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
SCOPES = ["https://www.googleapis.com/auth/youtube.readonly"]
|
||||||
|
CONFIG_DIR = os.environ.get("CONFIG_DIR", "/app/config")
|
||||||
|
_local_dir = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
_local_config = os.path.join(os.path.dirname(_local_dir), "config")
|
||||||
|
if os.path.exists(os.path.join(_local_config, "client_secret.json")):
|
||||||
|
CONFIG_DIR = _local_config
|
||||||
|
|
||||||
|
CLIENT_SECRET = os.path.join(CONFIG_DIR, "client_secret.json")
|
||||||
|
TOKEN_FILE = os.path.join(CONFIG_DIR, "token.pickle")
|
||||||
|
|
||||||
|
|
||||||
|
def get_credentials():
|
||||||
|
"""Load or refresh OAuth credentials, triggering browser flow if needed."""
|
||||||
|
creds = None
|
||||||
|
if os.path.exists(TOKEN_FILE):
|
||||||
|
with open(TOKEN_FILE, "rb") as token:
|
||||||
|
creds = pickle.load(token)
|
||||||
|
|
||||||
|
if not creds or not creds.valid:
|
||||||
|
if creds and creds.expired and creds.refresh_token:
|
||||||
|
creds.refresh(Request())
|
||||||
|
else:
|
||||||
|
if not os.path.exists(CLIENT_SECRET):
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"client_secret.json not found at {CLIENT_SECRET}. "
|
||||||
|
"Download from Google Cloud Console."
|
||||||
|
)
|
||||||
|
flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRET, SCOPES)
|
||||||
|
creds = flow.run_local_server(port=0)
|
||||||
|
|
||||||
|
os.makedirs(os.path.dirname(TOKEN_FILE), exist_ok=True)
|
||||||
|
with open(TOKEN_FILE, "wb") as token:
|
||||||
|
pickle.dump(creds, token)
|
||||||
|
|
||||||
|
return creds
|
||||||
+74
-62
@@ -1,97 +1,109 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import logging
|
import logging
|
||||||
import aiosqlite
|
import aiosqlite
|
||||||
from db import DB_PATH, init_db
|
from googleapiclient.discovery import build
|
||||||
|
from googleapiclient.errors import HttpError
|
||||||
|
from auth import get_credentials
|
||||||
|
from db import DB_PATH
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
COOKIES_PATH = os.environ.get("COOKIES_PATH", "/app/config/cookies.txt")
|
|
||||||
if not os.path.exists(COOKIES_PATH):
|
|
||||||
# Fallback to local directory
|
|
||||||
COOKIES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cookies.txt")
|
|
||||||
|
|
||||||
async def fetch_subscriptions():
|
async def fetch_subscriptions():
|
||||||
if not os.path.exists(COOKIES_PATH):
|
"""Fetch latest videos from subscribed channels using YouTube Data API v3."""
|
||||||
log.error(f"cookies.txt not found at {COOKIES_PATH}. Ingestion aborted.")
|
try:
|
||||||
|
creds = get_credentials()
|
||||||
|
except FileNotFoundError as e:
|
||||||
|
log.error(f"OAuth setup incomplete: {e}")
|
||||||
|
return 0
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f"OAuth credential error: {e}")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
log.info(f"Fetching subscriptions using cookies at {COOKIES_PATH}")
|
youtube = build("youtube", "v3", credentials=creds)
|
||||||
|
|
||||||
cmd = [
|
|
||||||
"yt-dlp",
|
|
||||||
"--flat-playlist",
|
|
||||||
"--dump-json",
|
|
||||||
"--cookies", COOKIES_PATH,
|
|
||||||
"--playlist-end", "40",
|
|
||||||
"--extractor-args", "youtubetab:skip=authcheck",
|
|
||||||
"https://www.youtube.com/feed/subscriptions"
|
|
||||||
]
|
|
||||||
|
|
||||||
|
# Step 1: Get subscribed channel IDs
|
||||||
|
channel_ids = []
|
||||||
try:
|
try:
|
||||||
process = await asyncio.create_subprocess_exec(
|
request = youtube.subscriptions().list(
|
||||||
*cmd,
|
part="snippet",
|
||||||
stdout=asyncio.subprocess.PIPE,
|
mine=True,
|
||||||
stderr=asyncio.subprocess.PIPE
|
maxResults=50,
|
||||||
|
order="alphabetical"
|
||||||
)
|
)
|
||||||
stdout, stderr = await process.communicate()
|
while request:
|
||||||
except Exception as e:
|
response = request.execute()
|
||||||
log.error(f"Failed to run yt-dlp: {e}")
|
for item in response.get("items", []):
|
||||||
|
cid = item["snippet"]["resourceId"]["channelId"]
|
||||||
|
if cid not in channel_ids:
|
||||||
|
channel_ids.append(cid)
|
||||||
|
request = youtube.subscriptions().list_next(request, response)
|
||||||
|
except HttpError as e:
|
||||||
|
log.error(f"YouTube API error fetching subscriptions: {e}")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
if process.returncode != 0:
|
log.info(f"Found {len(channel_ids)} subscribed channels")
|
||||||
log.error(f"yt-dlp error output: {stderr.decode('utf-8', errors='ignore')}")
|
|
||||||
|
if not channel_ids:
|
||||||
|
log.warning("No subscribed channels found. Make sure the authenticated account has subscriptions.")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
lines = stdout.decode('utf-8', errors='ignore').strip().split('\n')
|
# Step 2: For each channel, get latest videos
|
||||||
new_count = 0
|
new_count = 0
|
||||||
|
|
||||||
async with aiosqlite.connect(DB_PATH) as db:
|
async with aiosqlite.connect(DB_PATH) as db:
|
||||||
for line in lines:
|
for cid in channel_ids[:20]:
|
||||||
if not line:
|
|
||||||
continue
|
|
||||||
try:
|
try:
|
||||||
video = json.loads(line)
|
req = youtube.search().list(
|
||||||
video_id = video.get("id")
|
part="snippet",
|
||||||
if not video_id:
|
channelId=cid,
|
||||||
|
maxResults=10,
|
||||||
|
order="date",
|
||||||
|
type="video"
|
||||||
|
)
|
||||||
|
resp = req.execute()
|
||||||
|
for item in resp.get("items", []):
|
||||||
|
vid = item["id"]["videoId"]
|
||||||
|
snippet = item["snippet"]
|
||||||
|
|
||||||
|
cursor = await db.execute(
|
||||||
|
"SELECT 1 FROM videos WHERE video_id = ?", (vid,)
|
||||||
|
)
|
||||||
|
if await cursor.fetchone():
|
||||||
continue
|
continue
|
||||||
|
|
||||||
title = video.get("title", "Untitled")
|
pub_time = snippet.get("publishTime", "")
|
||||||
channel_name = video.get("uploader") or video.get("channel") or "Unknown Channel"
|
|
||||||
channel_id = video.get("uploader_id") or video.get("channel_id")
|
|
||||||
url = f"https://www.youtube.com/watch?v={video_id}"
|
|
||||||
thumbnail_url = video.get("thumbnail") or f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
|
|
||||||
|
|
||||||
raw_date = video.get("upload_date")
|
|
||||||
if raw_date and len(raw_date) == 8:
|
|
||||||
publish_date = f"{raw_date[:4]}-{raw_date[4:6]}-{raw_date[6:]}"
|
|
||||||
else:
|
|
||||||
publish_date = None
|
|
||||||
|
|
||||||
duration = video.get("duration")
|
|
||||||
|
|
||||||
# Check if video already exists in db
|
|
||||||
cursor = await db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,))
|
|
||||||
exists = await cursor.fetchone()
|
|
||||||
if not exists:
|
|
||||||
await db.execute("""
|
await db.execute("""
|
||||||
INSERT INTO videos (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration, status)
|
INSERT INTO videos
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending')
|
(video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date)
|
||||||
""", (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration))
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""", (
|
||||||
|
vid,
|
||||||
|
snippet.get("title", "Untitled"),
|
||||||
|
snippet.get("channelTitle", "Unknown"),
|
||||||
|
snippet.get("channelId", cid),
|
||||||
|
f"https://www.youtube.com/watch?v={vid}",
|
||||||
|
snippet.get("thumbnails", {}).get("high", {}).get("url", ""),
|
||||||
|
pub_time[:10] if pub_time else None
|
||||||
|
))
|
||||||
new_count += 1
|
new_count += 1
|
||||||
except Exception as e:
|
log.debug(f"New video: {snippet.get('title', 'Untitled')[:50]}")
|
||||||
log.warning(f"Error parsing yt-dlp line: {e}")
|
except HttpError as e:
|
||||||
|
log.warning(f"Channel {cid} fetch error: {e}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
log.info(f"Ingestion completed. Found {new_count} new videos.")
|
log.info(f"Ingestion completed. Found {new_count} new videos.")
|
||||||
return new_count
|
return new_count
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
import sys
|
import sys
|
||||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||||
|
|
||||||
async def main():
|
async def main():
|
||||||
|
from db import init_db
|
||||||
await init_db()
|
await init_db()
|
||||||
await fetch_subscriptions()
|
count = await fetch_subscriptions()
|
||||||
|
print(f"\nDone. {count} new videos ingested.")
|
||||||
|
|
||||||
asyncio.run(main())
|
asyncio.run(main())
|
||||||
|
|||||||
@@ -7,3 +7,6 @@ httpx>=0.24.0
|
|||||||
pytest>=7.0.0
|
pytest>=7.0.0
|
||||||
pytest-asyncio>=0.21.0
|
pytest-asyncio>=0.21.0
|
||||||
pytest-mock>=3.10.0
|
pytest-mock>=3.10.0
|
||||||
|
google-api-python-client>=2.140.0
|
||||||
|
google-auth-oauthlib>=1.2.0
|
||||||
|
google-auth-httplib2>=0.2.0
|
||||||
|
|||||||
+4
-2
@@ -7,12 +7,14 @@ services:
|
|||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
container_name: t-youtube
|
container_name: t-youtube
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
ports:
|
network_mode: host
|
||||||
- "8000:8000"
|
|
||||||
environment:
|
environment:
|
||||||
- OPENAI_API_KEY=${OPENAI_API_KEY:-sk-Kq5JbgmnEEkdzTqNAQ6bV6nztQ0ngeHTHF9Vg8nUglQvrOlx}
|
- OPENAI_API_KEY=${OPENAI_API_KEY:-sk-Kq5JbgmnEEkdzTqNAQ6bV6nztQ0ngeHTHF9Vg8nUglQvrOlx}
|
||||||
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
|
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
|
||||||
- ANTHROPIC_BASE_URL=https://www.bytecatcode.org
|
- ANTHROPIC_BASE_URL=https://www.bytecatcode.org
|
||||||
|
- HTTP_PROXY=http://100.70.115.1:8118
|
||||||
|
- HTTPS_PROXY=http://100.70.115.1:8118
|
||||||
|
- NO_PROXY=localhost,127.0.0.1,100.0.0.0/8,192.168.0.0/16
|
||||||
volumes:
|
volumes:
|
||||||
- ./data:/app/data
|
- ./data:/app/data
|
||||||
- ./config:/app/config
|
- ./config:/app/config
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Export YouTube cookies from Chrome. Run: ytc"""
|
||||||
|
|
||||||
|
import os, sys, sqlite3, subprocess, hashlib, shutil, tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
OUTPUT = REPO_ROOT / "config" / "cookies.txt"
|
||||||
|
COOKIES_DB = Path.home() / "Library/Application Support/Google/Chrome/Default/Cookies"
|
||||||
|
|
||||||
|
SALT = b'saltysalt'
|
||||||
|
KEYLEN = 16 # AES-128
|
||||||
|
PBKDF2_ITERS = 1003
|
||||||
|
|
||||||
|
|
||||||
|
def get_key():
|
||||||
|
r = subprocess.run(
|
||||||
|
['security', 'find-generic-password', '-w', '-s', 'Chrome Safe Storage'],
|
||||||
|
capture_output=True, text=True, timeout=30
|
||||||
|
)
|
||||||
|
if r.returncode != 0:
|
||||||
|
print("ERROR: Keychain access failed. Approve the dialog.", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
return hashlib.pbkdf2_hmac('sha1', r.stdout.strip().encode(), SALT, PBKDF2_ITERS, KEYLEN)
|
||||||
|
|
||||||
|
|
||||||
|
def decrypt(blob, key):
|
||||||
|
"""Decrypt Chrome v10 encrypted value. Returns str or None."""
|
||||||
|
if not blob or not blob.startswith(b'v10'):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||||
|
from cryptography.hazmat.primitives import padding
|
||||||
|
iv = blob[3:19]
|
||||||
|
ct = blob[19:]
|
||||||
|
c = Cipher(algorithms.AES(key), modes.CBC(iv))
|
||||||
|
d = c.decryptor()
|
||||||
|
pt = d.update(ct) + d.finalize()
|
||||||
|
u = padding.PKCS7(128).unpadder()
|
||||||
|
return (u.update(pt) + u.finalize()).decode('utf-8', errors='replace')
|
||||||
|
except Exception as e:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
print("Getting Chrome key from Keychain... (approve dialog)")
|
||||||
|
key = get_key()
|
||||||
|
|
||||||
|
# Copy DB to avoid lock
|
||||||
|
with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as f:
|
||||||
|
tmp = f.name
|
||||||
|
shutil.copy2(COOKIES_DB, tmp)
|
||||||
|
|
||||||
|
conn = sqlite3.connect(tmp)
|
||||||
|
conn.text_factory = bytes
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT host_key, name, encrypted_value, value, path, expires_utc, is_secure, is_httponly "
|
||||||
|
"FROM cookies WHERE host_key LIKE '%youtube.com' ORDER BY name"
|
||||||
|
).fetchall()
|
||||||
|
conn.close()
|
||||||
|
os.unlink(tmp)
|
||||||
|
|
||||||
|
print(f"Found {len(rows)} YouTube cookies")
|
||||||
|
|
||||||
|
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
ok, fail, skipped = 0, 0, 0
|
||||||
|
|
||||||
|
with open(OUTPUT, 'w') as f:
|
||||||
|
f.write("# Netscape HTTP Cookie File\n# Extracted from Chrome\n\n")
|
||||||
|
for hk, name, enc_val, raw_val, path, expires, secure, httponly in rows:
|
||||||
|
val = decrypt(enc_val, key)
|
||||||
|
if val is None and raw_val:
|
||||||
|
val = raw_val.decode('utf-8', errors='replace')
|
||||||
|
if not val:
|
||||||
|
n = name.decode() if isinstance(name, bytes) else name
|
||||||
|
print(f" SKIP: {n} (decrypt failed, no raw value)")
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
d = hk.decode() if isinstance(hk, bytes) else hk
|
||||||
|
n = name.decode() if isinstance(name, bytes) else name
|
||||||
|
p = path.decode() if isinstance(path, bytes) else path
|
||||||
|
flag = "TRUE" if d.startswith('.') else "FALSE"
|
||||||
|
s = "TRUE" if secure else "FALSE"
|
||||||
|
e = int(expires / 1_000_000) if expires and expires > 0 else 0
|
||||||
|
|
||||||
|
if httponly:
|
||||||
|
f.write(f"#HttpOnly_{d}\t{flag}\t{p}\t{s}\t{e}\t{n}\t{val}\n")
|
||||||
|
f.write(f"{d}\t{flag}\t{p}\t{s}\t{e}\t{n}\t{val}\n")
|
||||||
|
ok += 1
|
||||||
|
|
||||||
|
print(f"\nExported: {ok} Failed: {fail} Skipped: {skipped}")
|
||||||
|
print(f"Output: {OUTPUT} ({OUTPUT.stat().st_size} bytes)")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user