diff --git a/backend/fetcher.py b/backend/fetcher.py index 26030c8..d07ee93 100644 --- a/backend/fetcher.py +++ b/backend/fetcher.py @@ -1,5 +1,6 @@ import asyncio import os +import json import html import logging import aiosqlite @@ -14,6 +15,36 @@ log = logging.getLogger(__name__) ATOM_NS = "http://www.w3.org/2005/Atom" +CHANNELS_PER_SYNC = 50 # Rotate through all channels — N per sync +_ROTATION_FILE = os.path.join(os.environ.get("DB_DIR", os.path.dirname(os.path.abspath(__file__))), "sync_rotation.json") + + +def _get_rotation_slice(total: int) -> tuple[int, int]: + """Return (start, end) indices for this sync's channel slice.""" + state = {"offset": 0, "total": total} + if os.path.exists(_ROTATION_FILE): + try: + with open(_ROTATION_FILE) as f: + state = json.load(f) + except (json.JSONDecodeError, OSError): + pass + # If total changed (subscribed/unsubscribed), reset + if state.get("total") != total: + state = {"offset": 0, "total": total} + start = state["offset"] + end = min(start + CHANNELS_PER_SYNC, total) + return start, end + + +def _save_rotation_slice(total: int, start: int, end: int): + """Persist next sync offset.""" + next_offset = end # Continue from where we left off + if next_offset >= total: + next_offset = 0 # Wrap around — full cycle complete + os.makedirs(os.path.dirname(_ROTATION_FILE), exist_ok=True) + with open(_ROTATION_FILE, "w") as f: + json.dump({"offset": next_offset, "total": total}, f) + async def _fetch_rss(client, channel_id): """Fetch latest videos from a channel's RSS feed (free). Returns list of (video_id, title).""" url = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}" @@ -61,12 +92,17 @@ async def fetch_subscriptions(): log.info(f"Found {len(channel_ids)} subscribed channels") - # Step 2: Fetch RSS from all channels (free, parallel) + # Step 2: Rotate — only process CHANNELS_PER_SYNC channels per sync + start, end = _get_rotation_slice(len(channel_ids)) + slice_ids = channel_ids[start:end] + log.info(f"Processing channels {start+1}–{end} of {len(channel_ids)} ({len(slice_ids)} this sync)") + + # Step 3: Fetch RSS from slice channels (free, parallel) rss_videos = {} async with httpx.AsyncClient(timeout=20) as client: - tasks = [_fetch_rss(client, cid) for cid in channel_ids] + tasks = [_fetch_rss(client, cid) for cid in slice_ids] results = await asyncio.gather(*tasks) - for cid, entries in zip(channel_ids, results): + for cid, entries in zip(slice_ids, results): if entries: rss_videos[cid] = entries @@ -81,6 +117,7 @@ async def fetch_subscriptions(): log.info(f"RSS discovered {len(all_video_ids)} videos from {len(rss_videos)} channels") if not all_video_ids: + _save_rotation_slice(len(channel_ids), start, end) return 0 # Step 3: Check which videos are new (in SQLite) @@ -94,6 +131,7 @@ async def fetch_subscriptions(): log.info(f"{len(new_ids)} new videos to fetch metadata for") if not new_ids: + _save_rotation_slice(len(channel_ids), start, end) return 0 # Step 4: Fetch metadata via videos.list (1 unit per 50 videos) @@ -129,6 +167,10 @@ async def fetch_subscriptions(): await db.commit() log.info(f"Ingestion completed. Found {new_count} new videos.") + + # Save rotation state for next sync + _save_rotation_slice(len(channel_ids), start, end) + return new_count diff --git a/backend/summarizer.py b/backend/summarizer.py index c15e1a2..2cd8bb9 100644 --- a/backend/summarizer.py +++ b/backend/summarizer.py @@ -62,9 +62,9 @@ def download_transcript(video_id: str) -> str: log.warning(f"Failed to fetch transcript for {video_id}: {e}") return None -async def summarize_video(video_id: str) -> str: - # 1. Try transcript, fall back to description - transcript = download_transcript(video_id) +async def summarize_video(video_id: str) -> str | None: + # 1. Try transcript, fall back to description (offload sync call to thread) + transcript = await asyncio.to_thread(download_transcript, video_id) if not transcript: # Fallback: use video description from YouTube API @@ -120,7 +120,7 @@ async def summarize_video(video_id: str) -> str: summary = data["choices"][0]["message"]["content"] except Exception as e: log.error(f"DeepSeek API request failed for {video_id}: {e}") - summary = f"Error generating summary: {e}" + return None # Leave summary NULL for retry on next sync # 3. Save transcript & summary to db async with aiosqlite.connect(DB_PATH) as db: diff --git a/backend/tests/test_fetcher.py b/backend/tests/test_fetcher.py index 43a353a..6f73023 100644 --- a/backend/tests/test_fetcher.py +++ b/backend/tests/test_fetcher.py @@ -1,112 +1,186 @@ +"""Tests for fetcher.py — mocks YouTube Data API v3 + RSS.""" import pytest import os import json import aiosqlite import fetcher -import asyncio -from unittest.mock import patch, AsyncMock +from unittest.mock import patch, AsyncMock, MagicMock -class MockProcess: - def __init__(self, stdout_data, returncode=0, stderr_data=b""): - self.stdout_data = stdout_data - self.stderr_data = stderr_data - self.returncode = returncode - async def communicate(self): - return self.stdout_data, self.stderr_data +def _clean_rotation_file(): + """Remove rotation state file if it exists.""" + path = getattr(fetcher, "_ROTATION_FILE", None) + if path and os.path.exists(path): + os.remove(path) + + +@pytest.fixture(autouse=True) +def auto_clean_rotation(): + _clean_rotation_file() + yield + _clean_rotation_file() + + +def _mock_youtube_build(sub_channels=None, video_data=None): + """Build a mock YouTube API service that returns canned responses.""" + sub_channels = sub_channels or [ + {"snippet": {"resourceId": {"channelId": "chan001"}}}, + {"snippet": {"resourceId": {"channelId": "chan002"}}}, + ] + video_data = video_data or {} + + def mock_subscriptions_list(**kwargs): + return MagicMock(execute=MagicMock(return_value={"items": sub_channels})) + + def mock_videos_list(**kwargs): + vid_str = kwargs.get("id", "") + ids = [v for v in vid_str.split(",") if v] + items = [] + for vid in ids: + data = video_data.get(vid, { + "title": f"Video {vid}", + "channelTitle": "Test Channel", + "channelId": "chan001", + "thumbnails": {"high": {"url": f"https://img.youtube.com/vi/{vid}/hqdefault.jpg"}}, + "publishTime": "2026-06-01T00:00:00Z", + }) + items.append({"id": vid, "snippet": data}) + return MagicMock(execute=MagicMock(return_value={"items": items})) + + svc = MagicMock() + svc.subscriptions = MagicMock(return_value=MagicMock(list=mock_subscriptions_list, list_next=MagicMock(return_value=None))) + svc.videos = MagicMock(return_value=MagicMock(list=mock_videos_list)) + return svc + + +def _mock_rss_response(videos): + """Build RSS XML string for a list of (video_id, title) tuples.""" + entries = [] + for vid, title in videos: + entries.append(f""" + yt:video:{vid} + + {title} + """) + return f""" + + {''.join(entries)} +""" + @pytest.mark.asyncio -async def test_fetch_subscriptions_no_cookies(tmp_path, monkeypatch): - monkeypatch.setattr(fetcher, "COOKIES_PATH", "/non/existent/path/cookies.txt") - count = await fetcher.fetch_subscriptions() - assert count == 0 - -@pytest.mark.asyncio -async def test_fetch_subscriptions_success(test_db, tmp_path, monkeypatch): - cookies_file = os.path.join(tmp_path, "mock_cookies.txt") - with open(cookies_file, "w") as f: - f.write("# Netscape HTTP Cookie File") - monkeypatch.setattr(fetcher, "COOKIES_PATH", cookies_file) - - video1 = { - "id": "vid123", - "title": "Mock Video 1", - "uploader": "Uploader One", - "uploader_id": "chan123", - "upload_date": "20260601", - "duration": 360, - "thumbnail": "https://img.youtube.com/vi/vid123/hqdefault.jpg" - } - video2 = { - "id": "vid456", - "title": "Mock Video 2", - "uploader": "Uploader Two", - "uploader_id": "chan456", - "upload_date": "20260602", - "duration": 720 - } - stdout_bytes = (json.dumps(video1) + "\n" + json.dumps(video2) + "\n").encode("utf-8") - - mock_process = MockProcess(stdout_data=stdout_bytes, returncode=0) - - with patch("asyncio.create_subprocess_exec", return_value=mock_process) as mock_exec: +async def test_fetch_zero_channels(test_db): + """When no subscriptions exist, should return 0.""" + mock_svc = _mock_youtube_build(sub_channels=[]) + with patch("fetcher.build", return_value=mock_svc), \ + patch("fetcher.get_credentials", return_value="mock_creds"): count = await fetcher.fetch_subscriptions() + assert count == 0 - mock_exec.assert_called_once() - args = mock_exec.call_args[0] - assert "yt-dlp" in args - assert "https://www.youtube.com/feed/subscriptions" in args - assert cookies_file in args + +@pytest.mark.asyncio +async def test_fetch_rss_empty(test_db): + """When RSS returns nothing, count should be 0.""" + mock_svc = _mock_youtube_build() + with patch("fetcher.build", return_value=mock_svc), \ + patch("fetcher.get_credentials", return_value="mock_creds"), \ + patch("httpx.AsyncClient.get", AsyncMock(return_value=MagicMock(status_code=404))): + count = await fetcher.fetch_subscriptions() + assert count == 0 + + +@pytest.mark.asyncio +async def test_fetch_new_videos(test_db): + """New videos from RSS should be inserted into DB.""" + mock_svc = _mock_youtube_build(video_data={ + "vid001": {"title": "First Video", "channelTitle": "Chan One", + "channelId": "chan001", "thumbnails": {"high": {"url": ""}}, + "publishTime": "2026-06-01T00:00:00Z"}, + "vid002": {"title": "Second Video", "channelTitle": "Chan Two", + "channelId": "chan002", "thumbnails": {"high": {"url": ""}}, + "publishTime": "2026-06-02T00:00:00Z"}, + }) + rss_xml = _mock_rss_response([("vid001", "First Video"), ("vid002", "Second Video")]) + mock_response = MagicMock(status_code=200, text=rss_xml) + + with patch("fetcher.build", return_value=mock_svc), \ + patch("fetcher.get_credentials", return_value="mock_creds"), \ + patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)): + count = await fetcher.fetch_subscriptions() assert count == 2 - # Query db to verify values + # Verify DB cursor = await test_db.execute("SELECT * FROM videos ORDER BY video_id") - videos = await cursor.fetchall() - assert len(videos) == 2 + rows = await cursor.fetchall() + assert len(rows) == 2 + assert rows[0]["video_id"] == "vid001" + assert rows[0]["title"] == "First Video" + assert rows[1]["video_id"] == "vid002" + assert rows[1]["title"] == "Second Video" - assert videos[0]["video_id"] == "vid123" - assert videos[0]["title"] == "Mock Video 1" - assert videos[0]["channel_name"] == "Uploader One" - assert videos[0]["channel_id"] == "chan123" - assert videos[0]["url"] == "https://www.youtube.com/watch?v=vid123" - assert videos[0]["thumbnail_url"] == "https://img.youtube.com/vi/vid123/hqdefault.jpg" - assert videos[0]["publish_date"] == "2026-06-01" - assert videos[0]["duration"] == 360 - assert videos[0]["status"] == "pending" - - assert videos[1]["video_id"] == "vid456" - assert videos[1]["title"] == "Mock Video 2" - assert videos[1]["publish_date"] == "2026-06-02" - assert videos[1]["duration"] == 720 @pytest.mark.asyncio -async def test_fetch_subscriptions_skip_duplicates(test_db, tmp_path, monkeypatch): - cookies_file = os.path.join(tmp_path, "mock_cookies.txt") - with open(cookies_file, "w") as f: - f.write("# Netscape HTTP Cookie File") - monkeypatch.setattr(fetcher, "COOKIES_PATH", cookies_file) - - # Insert an existing video record first - await test_db.execute(""" - INSERT INTO videos (video_id, title, url, status) - VALUES ('vid123', 'Mock Video 1', 'https://www.youtube.com/watch?v=vid123', 'pending') - """) +async def test_fetch_skip_duplicates(test_db): + """Existing videos should be skipped.""" + # Pre-insert one video + await test_db.execute( + "INSERT INTO videos (video_id, title, url, status) VALUES ('vid001', 'Old', 'https://youtube.com/watch?v=vid001', 'pending')" + ) await test_db.commit() - video1 = { - "id": "vid123", - "title": "Mock Video 1", - "upload_date": "20260601" - } - video2 = { - "id": "vid456", - "title": "Mock Video 2", - "upload_date": "20260602" - } - stdout_bytes = (json.dumps(video1) + "\n" + json.dumps(video2) + "\n").encode("utf-8") + mock_svc = _mock_youtube_build(video_data={ + "vid001": {"title": "First Video (updated)", "channelTitle": "Chan One", + "channelId": "chan001"}, + "vid002": {"title": "Second Video", "channelTitle": "Chan Two", + "channelId": "chan002"}, + }) + rss_xml = _mock_rss_response([("vid001", "First Video"), ("vid002", "Second Video")]) + mock_response = MagicMock(status_code=200, text=rss_xml) - mock_process = MockProcess(stdout_data=stdout_bytes, returncode=0) - - with patch("asyncio.create_subprocess_exec", return_value=mock_process): + with patch("fetcher.build", return_value=mock_svc), \ + patch("fetcher.get_credentials", return_value="mock_creds"), \ + patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)): count = await fetcher.fetch_subscriptions() - assert count == 1 # Only 1 new video should be added (vid456) + assert count == 1 # Only vid002 is new + + # Verify DB — vid001 still has original title (not updated) + cursor = await test_db.execute("SELECT title FROM videos WHERE video_id='vid001'") + row = await cursor.fetchone() + assert row["title"] == "Old" + + +@pytest.mark.asyncio +async def test_rotation_advances(test_db): + """Rotation slice should advance after each sync.""" + # Mock 150 channels + sub_channels = [{"snippet": {"resourceId": {"channelId": f"chan{i:03d}"}}} for i in range(150)] + mock_svc = _mock_youtube_build(sub_channels=sub_channels) + + rss_xml = _mock_rss_response([]) # No videos, just test rotation + mock_response = MagicMock(status_code=200, text=rss_xml) + + with patch("fetcher.build", return_value=mock_svc), \ + patch("fetcher.get_credentials", return_value="mock_creds"), \ + patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)): + # First sync — should process chan000-chan049 + count = await fetcher.fetch_subscriptions() + # Second sync — chan050-chan099 + count2 = await fetcher.fetch_subscriptions() + # Third sync — chan100-chan149 + count3 = await fetcher.fetch_subscriptions() + # Fourth sync — wrap around, chan000-chan049 again + count4 = await fetcher.fetch_subscriptions() + + # Verify rotation file + with open(fetcher._ROTATION_FILE) as f: + state = json.load(f) + assert state["offset"] == 50 # After 4th sync, offset = 50 (wrapped from 150) + assert state["total"] == 150 + + +@pytest.mark.asyncio +async def test_fetch_quietly_handles_missing_credentials(test_db): + """When OAuth is missing, should return 0 gracefully.""" + with patch("fetcher.get_credentials", side_effect=FileNotFoundError("No client_secret")): + count = await fetcher.fetch_subscriptions() + assert count == 0 diff --git a/backend/tests/test_summarizer.py b/backend/tests/test_summarizer.py index 6e25c86..cd98786 100644 --- a/backend/tests/test_summarizer.py +++ b/backend/tests/test_summarizer.py @@ -1,3 +1,4 @@ +"""Tests for summarizer.py — mocks transcript API + DeepSeek API.""" import pytest import os import httpx @@ -5,6 +6,7 @@ import aiosqlite import summarizer from unittest.mock import patch, MagicMock, AsyncMock + class MockTranscript: def fetch(self): return [ @@ -25,6 +27,17 @@ class MockTranscriptList: def __iter__(self): return iter([self.transcript]) + +def _mock_deepseek_response(content: str) -> MagicMock: + """Build a DeepSeek/OpenAI-compatible API response.""" + r = MagicMock(spec=httpx.Response) + r.status_code = 200 + r.json.return_value = { + "choices": [{"message": {"content": content}}] + } + return r + + @pytest.mark.asyncio async def test_download_transcript_success(): mock_trans = MockTranscript() @@ -34,9 +47,9 @@ async def test_download_transcript_success(): text = summarizer.download_transcript("test_vid") assert text == "Welcome to this video tutorial. Today we are learning about Svelte 5 and runes." + @pytest.mark.asyncio async def test_summarize_video_success(test_db): - # Insert a pending video in db first await test_db.execute(""" INSERT INTO videos (video_id, title, url, status) VALUES ('summarize_vid', 'Video to Summarize', 'https://www.youtube.com/watch?v=summarize_vid', 'pending') @@ -45,19 +58,8 @@ async def test_summarize_video_success(test_db): mock_trans = MockTranscript() mock_list = MockTranscriptList(mock_trans) + mock_response = _mock_deepseek_response("**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line.") - # Mock Claude HTTP response - mock_response = MagicMock(spec=httpx.Response) - mock_response.status_code = 200 - mock_response.json.return_value = { - "content": [ - { - "text": "**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line." - } - ] - } - - # Patch both transcript api and httpx AsyncClient post call with patch("youtube_transcript_api.YouTubeTranscriptApi.list", return_value=mock_list), \ patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response) as mock_post: @@ -69,12 +71,12 @@ async def test_summarize_video_success(test_db): assert summarizer.BASE_URL in url headers = mock_post.call_args[1]["headers"] - assert "x-api-key" in headers + assert "Authorization" in headers assert headers["User-Agent"] is not None payload = mock_post.call_args[1]["json"] - assert payload["model"] == "claude-haiku-4-5-20251001" - assert len(payload["messages"]) == 1 + assert payload["model"] == "deepseek-chat" + assert len(payload["messages"]) == 2 # system + user assert summary == "**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line." @@ -85,9 +87,40 @@ async def test_summarize_video_success(test_db): assert video["transcript"] == "Welcome to this video tutorial. Today we are learning about Svelte 5 and runes." assert video["summary"] == "**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line." + +@pytest.mark.asyncio +async def test_summarize_video_deepseek_error_leaves_null(test_db): + """When DeepSeek fails, summary should stay NULL for retry.""" + await test_db.execute(""" + INSERT INTO videos (video_id, title, url, status) + VALUES ('err_vid', 'Error Video', 'https://youtube.com/watch?v=err_vid', 'pending') + """) + await test_db.commit() + + mock_trans = MockTranscript() + mock_list = MockTranscriptList(mock_trans) + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 429 + # raise_for_status on 429 should raise HTTPStatusError + mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( + "429 Rate Limited", request=MagicMock(), response=mock_response + ) + + with patch("youtube_transcript_api.YouTubeTranscriptApi.list", return_value=mock_list), \ + patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response): + + result = await summarizer.summarize_video("err_vid") + + assert result is None # Returns None instead of an error string + + # Verify DB — summary should remain NULL + cursor = await test_db.execute("SELECT summary FROM videos WHERE video_id = 'err_vid'") + row = await cursor.fetchone() + assert row["summary"] is None + + @pytest.mark.asyncio async def test_summarize_all_pending(test_db): - # Insert multiple videos, one pending and one already summarized await test_db.execute(""" INSERT INTO videos (video_id, title, url, summary, status) VALUES @@ -98,28 +131,21 @@ async def test_summarize_all_pending(test_db): mock_trans = MockTranscript() mock_list = MockTranscriptList(mock_trans) - - mock_response = MagicMock(spec=httpx.Response) - mock_response.status_code = 200 - mock_response.json.return_value = { - "content": [{"text": "Fresh summary"}] - } + mock_response = _mock_deepseek_response("Fresh summary") with patch("youtube_transcript_api.YouTubeTranscriptApi.list", return_value=mock_list), \ patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response) as mock_post: await summarizer.summarize_all_pending() - # Should only call API once for 'pending_vid' + # Should only call API once for 'pending_vid' (done_vid already has summary) mock_post.assert_called_once() - payload = mock_post.call_args[1]["json"] - assert "Welcome to this video tutorial." in payload["messages"][0]["content"] # Verify db states cursor = await test_db.execute("SELECT video_id, summary FROM videos ORDER BY video_id") videos = await cursor.fetchall() assert videos[0]["video_id"] == "done_vid" - assert videos[0]["summary"] == "Existing summary" # Untouched + assert videos[0]["summary"] == "Existing summary" # Untouched assert videos[1]["video_id"] == "pending_vid" - assert videos[1]["summary"] == "Fresh summary" # Updated + assert videos[1]["summary"] == "Fresh summary" # Updated diff --git a/docker-compose.yml b/docker-compose.yml index 5a069dd..9c1bc9c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,8 @@ services: restart: unless-stopped ports: - "8001:8000" + env_file: + - .env environment: - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY} - DEEPSEEK_BASE_URL=https://api.deepseek.com diff --git a/frontend/src/routes/YoutubeDashboard.svelte b/frontend/src/routes/YoutubeDashboard.svelte index fc1cacc..145a8d4 100644 --- a/frontend/src/routes/YoutubeDashboard.svelte +++ b/frontend/src/routes/YoutubeDashboard.svelte @@ -88,8 +88,14 @@ syncMessage = 'Syncing...'; try { const res = await fetch('/api/videos/fetch', { method: 'POST' }); - if (res.ok) setTimeout(() => { loadVideos(); isSyncing = false; syncMessage = ''; }, 6000); - } catch (err) { isSyncing = false; syncMessage = ''; } + if (res.ok) { + syncMessage = 'Sync started...'; + setTimeout(() => { loadVideos(); isSyncing = false; syncMessage = ''; }, 6000); + } else { + isSyncing = false; + syncMessage = 'Sync failed'; + } + } catch (err) { isSyncing = false; syncMessage = 'Network error'; } } async function updateStatus(videoId, newStatus, silent = false) { @@ -217,8 +223,12 @@ function saveNote() { const vid = selectedVideo?.video_id; if (!vid) return; - const text = notes[vid] || ''; - const stamped = text && !text.startsWith('[') ? `[${new Date().toLocaleString()}] ${text}` : text; + const prev = notes[vid] || ''; + const textarea = document.querySelector('#note-textarea'); + const current = textarea ? textarea.value : prev; + if (!current.trim()) return; + const ts = new Date().toLocaleString(); + const stamped = `[${ts}] ${current.trim()}\n${prev}`; notes = {...notes, [vid]: stamped}; localStorage.setItem('t-yt-notes', JSON.stringify(notes)); activeNote = null; @@ -377,7 +387,7 @@ {#if activeNote === selectedVideo.video_id}
-