Initial commit: Add T YouTube text-based reader web app
- FastAPI backend running on port 8000 - Svelte 5 runes frontend dashboard supporting Read/Skip state toggles - YouTube subscription feed scraper using yt-dlp & Netscape cookies.txt - Transcript downloader and Claude 3.5 Haiku summarizer via bytecatcode proxy - Multi-stage Dockerfile and docker-compose.yml configuration Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Gan, Jimmy
@@ -0,0 +1,34 @@
|
||||
import os
|
||||
import aiosqlite
|
||||
|
||||
DB_DIR = os.environ.get("DB_DIR", os.path.dirname(os.path.abspath(__file__)))
|
||||
DB_PATH = os.path.join(DB_DIR, "t_youtube.db")
|
||||
|
||||
async def init_db():
|
||||
os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS videos (
|
||||
video_id TEXT PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
channel_name TEXT,
|
||||
channel_id TEXT,
|
||||
url TEXT NOT NULL,
|
||||
thumbnail_url TEXT,
|
||||
publish_date TEXT,
|
||||
duration INTEGER,
|
||||
transcript TEXT,
|
||||
summary TEXT,
|
||||
status TEXT DEFAULT 'pending',
|
||||
created_at TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
await db.commit()
|
||||
|
||||
async def get_db():
|
||||
db = await aiosqlite.connect(DB_PATH)
|
||||
db.row_factory = aiosqlite.Row
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
await db.close()
|
||||
@@ -0,0 +1,96 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import logging
|
||||
import aiosqlite
|
||||
from db import DB_PATH, init_db
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
COOKIES_PATH = os.environ.get("COOKIES_PATH", "/app/config/cookies.txt")
|
||||
if not os.path.exists(COOKIES_PATH):
|
||||
# Fallback to local directory
|
||||
COOKIES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cookies.txt")
|
||||
|
||||
async def fetch_subscriptions():
|
||||
if not os.path.exists(COOKIES_PATH):
|
||||
log.error(f"cookies.txt not found at {COOKIES_PATH}. Ingestion aborted.")
|
||||
return 0
|
||||
|
||||
log.info(f"Fetching subscriptions using cookies at {COOKIES_PATH}")
|
||||
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
"--flat-playlist",
|
||||
"--dump-json",
|
||||
"--cookies", COOKIES_PATH,
|
||||
"--playlist-end", "40",
|
||||
"https://www.youtube.com/feed/subscriptions"
|
||||
]
|
||||
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE
|
||||
)
|
||||
stdout, stderr = await process.communicate()
|
||||
except Exception as e:
|
||||
log.error(f"Failed to run yt-dlp: {e}")
|
||||
return 0
|
||||
|
||||
if process.returncode != 0:
|
||||
log.error(f"yt-dlp error output: {stderr.decode('utf-8', errors='ignore')}")
|
||||
return 0
|
||||
|
||||
lines = stdout.decode('utf-8', errors='ignore').strip().split('\n')
|
||||
new_count = 0
|
||||
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
for line in lines:
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
video = json.loads(line)
|
||||
video_id = video.get("id")
|
||||
if not video_id:
|
||||
continue
|
||||
|
||||
title = video.get("title", "Untitled")
|
||||
channel_name = video.get("uploader") or video.get("channel") or "Unknown Channel"
|
||||
channel_id = video.get("uploader_id") or video.get("channel_id")
|
||||
url = f"https://www.youtube.com/watch?v={video_id}"
|
||||
thumbnail_url = video.get("thumbnail") or f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
|
||||
|
||||
raw_date = video.get("upload_date")
|
||||
if raw_date and len(raw_date) == 8:
|
||||
publish_date = f"{raw_date[:4]}-{raw_date[4:6]}-{raw_date[6:]}"
|
||||
else:
|
||||
publish_date = None
|
||||
|
||||
duration = video.get("duration")
|
||||
|
||||
# Check if video already exists in db
|
||||
cursor = await db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,))
|
||||
exists = await cursor.fetchone()
|
||||
if not exists:
|
||||
await db.execute("""
|
||||
INSERT INTO videos (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending')
|
||||
""", (video_id, title, channel_name, channel_id, url, thumbnail_url, publish_date, duration))
|
||||
new_count += 1
|
||||
except Exception as e:
|
||||
log.warning(f"Error parsing yt-dlp line: {e}")
|
||||
continue
|
||||
await db.commit()
|
||||
|
||||
log.info(f"Ingestion completed. Found {new_count} new videos.")
|
||||
return new_count
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
async def main():
|
||||
await init_db()
|
||||
await fetch_subscriptions()
|
||||
asyncio.run(main())
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
import os
|
||||
import logging
|
||||
from fastapi import FastAPI, BackgroundTasks, Depends, HTTPException, Query
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, List
|
||||
import aiosqlite
|
||||
|
||||
from db import init_db, get_db
|
||||
from fetcher import fetch_subscriptions
|
||||
from summarizer import summarize_video, summarize_all_pending
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
app = FastAPI(title="T YouTube", description="Text-based YouTube Summarizer")
|
||||
|
||||
# Enable CORS for development
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
class StatusUpdate(BaseModel):
|
||||
status: str
|
||||
|
||||
# Database Startup Hook
|
||||
@app.on_event("startup")
|
||||
async def startup():
|
||||
await init_db()
|
||||
|
||||
@app.get("/api/health")
|
||||
async def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get("/api/videos")
|
||||
async def get_videos(
|
||||
status: Optional[str] = "pending",
|
||||
limit: int = 50,
|
||||
db: aiosqlite.Connection = Depends(get_db)
|
||||
):
|
||||
query = "SELECT * FROM videos WHERE status = ? ORDER BY publish_date DESC, created_at DESC LIMIT ?"
|
||||
cursor = await db.execute(query, (status, limit))
|
||||
rows = await cursor.fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def bg_fetch_and_summarize():
|
||||
log.info("Starting background subscription fetch and summarization loop...")
|
||||
try:
|
||||
new_count = await fetch_subscriptions()
|
||||
if new_count > 0:
|
||||
await summarize_all_pending()
|
||||
except Exception as e:
|
||||
log.error(f"Error in background fetch and summarize task: {e}")
|
||||
|
||||
@app.post("/api/videos/fetch")
|
||||
async def trigger_fetch(background_tasks: BackgroundTasks):
|
||||
background_tasks.add_task(bg_fetch_and_summarize)
|
||||
return {"message": "Subscription sync and summarization running in background."}
|
||||
|
||||
@app.post("/api/videos/{video_id}/summarize")
|
||||
async def trigger_summarize(video_id: str, background_tasks: BackgroundTasks, db: aiosqlite.Connection = Depends(get_db)):
|
||||
cursor = await db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,))
|
||||
exists = await cursor.fetchone()
|
||||
if not exists:
|
||||
raise HTTPException(status_code=404, detail="Video not found")
|
||||
|
||||
background_tasks.add_task(summarize_video, video_id)
|
||||
return {"message": f"Summarization for video {video_id} queued."}
|
||||
|
||||
@app.post("/api/videos/{video_id}/status")
|
||||
async def update_status(
|
||||
video_id: str,
|
||||
update: StatusUpdate,
|
||||
db: aiosqlite.Connection = Depends(get_db)
|
||||
):
|
||||
if update.status not in ["pending", "read", "skipped"]:
|
||||
raise HTTPException(status_code=400, detail="Invalid status value")
|
||||
|
||||
cursor = await db.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,))
|
||||
exists = await cursor.fetchone()
|
||||
if not exists:
|
||||
raise HTTPException(status_code=404, detail="Video not found")
|
||||
|
||||
await db.execute("UPDATE videos SET status = ? WHERE video_id = ?", (update.status, video_id))
|
||||
await db.commit()
|
||||
return {"message": f"Video {video_id} status updated to {update.status}."}
|
||||
|
||||
# Mount frontend built files for production if they exist
|
||||
frontend_build_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "frontend/dist")
|
||||
if os.path.exists(frontend_build_path):
|
||||
app.mount("/", StaticFiles(directory=frontend_build_path, html=True), name="frontend")
|
||||
log.info("Mounted static frontend build successfully.")
|
||||
else:
|
||||
log.warning("Frontend build directory 'frontend/dist' not found. API only mode is active.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
@@ -0,0 +1,6 @@
|
||||
fastapi>=0.100.0
|
||||
uvicorn>=0.20.0
|
||||
yt-dlp>=2023.0.0
|
||||
youtube-transcript-api>=0.6.0
|
||||
aiosqlite>=0.18.0
|
||||
httpx>=0.24.0
|
||||
@@ -0,0 +1,140 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import httpx
|
||||
import aiosqlite
|
||||
from youtube_transcript_api import YouTubeTranscriptApi
|
||||
from db import DB_PATH
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Fallback pattern to retrieve the correct bytecatcode credentials
|
||||
API_KEY = os.environ.get("ANTHROPIC_API_KEY")
|
||||
if not API_KEY:
|
||||
API_KEY = os.environ.get("OPENAI_API_KEY", "sk-Kq5JbgmnEEkdzTqNAQ6bV6nztQ0ngeHTHF9Vg8nUglQvrOlx")
|
||||
|
||||
BASE_URL = os.environ.get("ANTHROPIC_BASE_URL", "https://www.bytecatcode.org")
|
||||
MODEL = "claude-haiku-4-5-20251001"
|
||||
|
||||
SYSTEM_PROMPT = "You are a professional video summarizing assistant. Create highly concise, structured summaries from video transcripts. Your output must be in Chinese if the transcript is Chinese, otherwise in English."
|
||||
|
||||
PROMPT = """Read the following YouTube video transcript and write a super-concise, structured summary.
|
||||
|
||||
Guidelines:
|
||||
1. **Core Hook (1 sentence in bold)**: Explain exactly what this video is about and its main conclusion or thesis.
|
||||
2. **Key Takeaways (max 4-5 bullet points)**: Summarize the most important points, arguments, or steps. Focus on information density. Skip fluff, intros, sponsor segments, and generic remarks.
|
||||
3. **Actionable Idea / Final Verdict**: A single line explaining who this is for or a key action item.
|
||||
|
||||
Keep the entire summary under 150 words. Be extremely direct so I can decide in 10 seconds if I should watch the full video or skip it. Use the same language as the transcript (likely Chinese or English).
|
||||
|
||||
Transcript:
|
||||
{transcript}"""
|
||||
|
||||
def download_transcript(video_id: str) -> str:
|
||||
try:
|
||||
transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
|
||||
|
||||
# Try to find manual transcripts in preferred languages
|
||||
try:
|
||||
transcript = transcript_list.find_manually_created_transcript(['en', 'zh-Hans', 'zh-Hant', 'zh', 'ja'])
|
||||
except Exception:
|
||||
# Fallback to generated (auto-captions) in preferred languages
|
||||
try:
|
||||
transcript = transcript_list.find_generated_transcript(['en', 'zh-Hans', 'zh-Hant', 'zh', 'ja'])
|
||||
except Exception:
|
||||
# Fallback to any transcript
|
||||
transcript = next(iter(transcript_list))
|
||||
|
||||
parts = transcript.fetch()
|
||||
text = " ".join([part["text"] for part in parts])
|
||||
return text
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to fetch transcript for {video_id}: {e}")
|
||||
return None
|
||||
|
||||
async def summarize_video(video_id: str) -> str:
|
||||
# 1. Fetch transcript
|
||||
transcript = download_transcript(video_id)
|
||||
if not transcript:
|
||||
# Save placeholder transcript / status
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
await db.execute(
|
||||
"UPDATE videos SET transcript = ?, summary = ? WHERE video_id = ?",
|
||||
("No transcript available", "Could not retrieve video transcript.", video_id)
|
||||
)
|
||||
await db.commit()
|
||||
return None
|
||||
|
||||
# 2. Call Claude API
|
||||
headers = {
|
||||
"x-api-key": API_KEY,
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json",
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
}
|
||||
|
||||
# Standard Anthropic payload format
|
||||
payload = {
|
||||
"model": MODEL,
|
||||
"max_tokens": 1000,
|
||||
"system": SYSTEM_PROMPT,
|
||||
"messages": [
|
||||
{"role": "user", "content": PROMPT.format(transcript=transcript[:50000])}
|
||||
]
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
try:
|
||||
resp = await client.post(
|
||||
f"{BASE_URL}/v1/messages",
|
||||
headers=headers,
|
||||
json=payload
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
summary = data["content"][0]["text"]
|
||||
except Exception as e:
|
||||
log.error(f"Claude API request failed for {video_id}: {e}")
|
||||
summary = f"Error generating summary: {e}"
|
||||
|
||||
# 3. Save transcript & summary to db
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
await db.execute(
|
||||
"UPDATE videos SET transcript = ?, summary = ? WHERE video_id = ?",
|
||||
(transcript, summary, video_id)
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
return summary
|
||||
|
||||
async def summarize_all_pending():
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
cursor = await db.execute("SELECT video_id FROM videos WHERE summary IS NULL OR summary = ''")
|
||||
rows = await cursor.fetchall()
|
||||
|
||||
if not rows:
|
||||
log.info("No pending videos to summarize.")
|
||||
return
|
||||
|
||||
log.info(f"Found {len(rows)} pending videos to summarize.")
|
||||
for row in rows:
|
||||
video_id = row["video_id"]
|
||||
log.info(f"Summarizing video: {video_id}")
|
||||
await summarize_video(video_id)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
|
||||
async def main():
|
||||
if len(sys.argv) > 1:
|
||||
video_id = sys.argv[1]
|
||||
print(f"Summarizing single video: {video_id}")
|
||||
summary = await summarize_video(video_id)
|
||||
print(f"Summary:\n{summary}")
|
||||
else:
|
||||
await summarize_all_pending()
|
||||
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user