sprint: fixes batch — saveNote history, DeepSeek retry, channel rotation, test fixes, docker env, blocking transcript, sync errors
This commit is contained in:
Gan, Jimmy
+165
-91
@@ -1,112 +1,186 @@
|
||||
"""Tests for fetcher.py — mocks YouTube Data API v3 + RSS."""
|
||||
import pytest
|
||||
import os
|
||||
import json
|
||||
import aiosqlite
|
||||
import fetcher
|
||||
import asyncio
|
||||
from unittest.mock import patch, AsyncMock
|
||||
from unittest.mock import patch, AsyncMock, MagicMock
|
||||
|
||||
class MockProcess:
|
||||
def __init__(self, stdout_data, returncode=0, stderr_data=b""):
|
||||
self.stdout_data = stdout_data
|
||||
self.stderr_data = stderr_data
|
||||
self.returncode = returncode
|
||||
|
||||
async def communicate(self):
|
||||
return self.stdout_data, self.stderr_data
|
||||
def _clean_rotation_file():
|
||||
"""Remove rotation state file if it exists."""
|
||||
path = getattr(fetcher, "_ROTATION_FILE", None)
|
||||
if path and os.path.exists(path):
|
||||
os.remove(path)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def auto_clean_rotation():
|
||||
_clean_rotation_file()
|
||||
yield
|
||||
_clean_rotation_file()
|
||||
|
||||
|
||||
def _mock_youtube_build(sub_channels=None, video_data=None):
|
||||
"""Build a mock YouTube API service that returns canned responses."""
|
||||
sub_channels = sub_channels or [
|
||||
{"snippet": {"resourceId": {"channelId": "chan001"}}},
|
||||
{"snippet": {"resourceId": {"channelId": "chan002"}}},
|
||||
]
|
||||
video_data = video_data or {}
|
||||
|
||||
def mock_subscriptions_list(**kwargs):
|
||||
return MagicMock(execute=MagicMock(return_value={"items": sub_channels}))
|
||||
|
||||
def mock_videos_list(**kwargs):
|
||||
vid_str = kwargs.get("id", "")
|
||||
ids = [v for v in vid_str.split(",") if v]
|
||||
items = []
|
||||
for vid in ids:
|
||||
data = video_data.get(vid, {
|
||||
"title": f"Video {vid}",
|
||||
"channelTitle": "Test Channel",
|
||||
"channelId": "chan001",
|
||||
"thumbnails": {"high": {"url": f"https://img.youtube.com/vi/{vid}/hqdefault.jpg"}},
|
||||
"publishTime": "2026-06-01T00:00:00Z",
|
||||
})
|
||||
items.append({"id": vid, "snippet": data})
|
||||
return MagicMock(execute=MagicMock(return_value={"items": items}))
|
||||
|
||||
svc = MagicMock()
|
||||
svc.subscriptions = MagicMock(return_value=MagicMock(list=mock_subscriptions_list, list_next=MagicMock(return_value=None)))
|
||||
svc.videos = MagicMock(return_value=MagicMock(list=mock_videos_list))
|
||||
return svc
|
||||
|
||||
|
||||
def _mock_rss_response(videos):
|
||||
"""Build RSS XML string for a list of (video_id, title) tuples."""
|
||||
entries = []
|
||||
for vid, title in videos:
|
||||
entries.append(f"""<entry>
|
||||
<id>yt:video:{vid}</id>
|
||||
<link href="https://www.youtube.com/watch?v={vid}"/>
|
||||
<title>{title}</title>
|
||||
</entry>""")
|
||||
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
{''.join(entries)}
|
||||
</feed>"""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_subscriptions_no_cookies(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(fetcher, "COOKIES_PATH", "/non/existent/path/cookies.txt")
|
||||
count = await fetcher.fetch_subscriptions()
|
||||
assert count == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_subscriptions_success(test_db, tmp_path, monkeypatch):
|
||||
cookies_file = os.path.join(tmp_path, "mock_cookies.txt")
|
||||
with open(cookies_file, "w") as f:
|
||||
f.write("# Netscape HTTP Cookie File")
|
||||
monkeypatch.setattr(fetcher, "COOKIES_PATH", cookies_file)
|
||||
|
||||
video1 = {
|
||||
"id": "vid123",
|
||||
"title": "Mock Video 1",
|
||||
"uploader": "Uploader One",
|
||||
"uploader_id": "chan123",
|
||||
"upload_date": "20260601",
|
||||
"duration": 360,
|
||||
"thumbnail": "https://img.youtube.com/vi/vid123/hqdefault.jpg"
|
||||
}
|
||||
video2 = {
|
||||
"id": "vid456",
|
||||
"title": "Mock Video 2",
|
||||
"uploader": "Uploader Two",
|
||||
"uploader_id": "chan456",
|
||||
"upload_date": "20260602",
|
||||
"duration": 720
|
||||
}
|
||||
stdout_bytes = (json.dumps(video1) + "\n" + json.dumps(video2) + "\n").encode("utf-8")
|
||||
|
||||
mock_process = MockProcess(stdout_data=stdout_bytes, returncode=0)
|
||||
|
||||
with patch("asyncio.create_subprocess_exec", return_value=mock_process) as mock_exec:
|
||||
async def test_fetch_zero_channels(test_db):
|
||||
"""When no subscriptions exist, should return 0."""
|
||||
mock_svc = _mock_youtube_build(sub_channels=[])
|
||||
with patch("fetcher.build", return_value=mock_svc), \
|
||||
patch("fetcher.get_credentials", return_value="mock_creds"):
|
||||
count = await fetcher.fetch_subscriptions()
|
||||
assert count == 0
|
||||
|
||||
mock_exec.assert_called_once()
|
||||
args = mock_exec.call_args[0]
|
||||
assert "yt-dlp" in args
|
||||
assert "https://www.youtube.com/feed/subscriptions" in args
|
||||
assert cookies_file in args
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_rss_empty(test_db):
|
||||
"""When RSS returns nothing, count should be 0."""
|
||||
mock_svc = _mock_youtube_build()
|
||||
with patch("fetcher.build", return_value=mock_svc), \
|
||||
patch("fetcher.get_credentials", return_value="mock_creds"), \
|
||||
patch("httpx.AsyncClient.get", AsyncMock(return_value=MagicMock(status_code=404))):
|
||||
count = await fetcher.fetch_subscriptions()
|
||||
assert count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_new_videos(test_db):
|
||||
"""New videos from RSS should be inserted into DB."""
|
||||
mock_svc = _mock_youtube_build(video_data={
|
||||
"vid001": {"title": "First Video", "channelTitle": "Chan One",
|
||||
"channelId": "chan001", "thumbnails": {"high": {"url": ""}},
|
||||
"publishTime": "2026-06-01T00:00:00Z"},
|
||||
"vid002": {"title": "Second Video", "channelTitle": "Chan Two",
|
||||
"channelId": "chan002", "thumbnails": {"high": {"url": ""}},
|
||||
"publishTime": "2026-06-02T00:00:00Z"},
|
||||
})
|
||||
rss_xml = _mock_rss_response([("vid001", "First Video"), ("vid002", "Second Video")])
|
||||
mock_response = MagicMock(status_code=200, text=rss_xml)
|
||||
|
||||
with patch("fetcher.build", return_value=mock_svc), \
|
||||
patch("fetcher.get_credentials", return_value="mock_creds"), \
|
||||
patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)):
|
||||
count = await fetcher.fetch_subscriptions()
|
||||
assert count == 2
|
||||
|
||||
# Query db to verify values
|
||||
# Verify DB
|
||||
cursor = await test_db.execute("SELECT * FROM videos ORDER BY video_id")
|
||||
videos = await cursor.fetchall()
|
||||
assert len(videos) == 2
|
||||
rows = await cursor.fetchall()
|
||||
assert len(rows) == 2
|
||||
assert rows[0]["video_id"] == "vid001"
|
||||
assert rows[0]["title"] == "First Video"
|
||||
assert rows[1]["video_id"] == "vid002"
|
||||
assert rows[1]["title"] == "Second Video"
|
||||
|
||||
assert videos[0]["video_id"] == "vid123"
|
||||
assert videos[0]["title"] == "Mock Video 1"
|
||||
assert videos[0]["channel_name"] == "Uploader One"
|
||||
assert videos[0]["channel_id"] == "chan123"
|
||||
assert videos[0]["url"] == "https://www.youtube.com/watch?v=vid123"
|
||||
assert videos[0]["thumbnail_url"] == "https://img.youtube.com/vi/vid123/hqdefault.jpg"
|
||||
assert videos[0]["publish_date"] == "2026-06-01"
|
||||
assert videos[0]["duration"] == 360
|
||||
assert videos[0]["status"] == "pending"
|
||||
|
||||
assert videos[1]["video_id"] == "vid456"
|
||||
assert videos[1]["title"] == "Mock Video 2"
|
||||
assert videos[1]["publish_date"] == "2026-06-02"
|
||||
assert videos[1]["duration"] == 720
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_subscriptions_skip_duplicates(test_db, tmp_path, monkeypatch):
|
||||
cookies_file = os.path.join(tmp_path, "mock_cookies.txt")
|
||||
with open(cookies_file, "w") as f:
|
||||
f.write("# Netscape HTTP Cookie File")
|
||||
monkeypatch.setattr(fetcher, "COOKIES_PATH", cookies_file)
|
||||
|
||||
# Insert an existing video record first
|
||||
await test_db.execute("""
|
||||
INSERT INTO videos (video_id, title, url, status)
|
||||
VALUES ('vid123', 'Mock Video 1', 'https://www.youtube.com/watch?v=vid123', 'pending')
|
||||
""")
|
||||
async def test_fetch_skip_duplicates(test_db):
|
||||
"""Existing videos should be skipped."""
|
||||
# Pre-insert one video
|
||||
await test_db.execute(
|
||||
"INSERT INTO videos (video_id, title, url, status) VALUES ('vid001', 'Old', 'https://youtube.com/watch?v=vid001', 'pending')"
|
||||
)
|
||||
await test_db.commit()
|
||||
|
||||
video1 = {
|
||||
"id": "vid123",
|
||||
"title": "Mock Video 1",
|
||||
"upload_date": "20260601"
|
||||
}
|
||||
video2 = {
|
||||
"id": "vid456",
|
||||
"title": "Mock Video 2",
|
||||
"upload_date": "20260602"
|
||||
}
|
||||
stdout_bytes = (json.dumps(video1) + "\n" + json.dumps(video2) + "\n").encode("utf-8")
|
||||
mock_svc = _mock_youtube_build(video_data={
|
||||
"vid001": {"title": "First Video (updated)", "channelTitle": "Chan One",
|
||||
"channelId": "chan001"},
|
||||
"vid002": {"title": "Second Video", "channelTitle": "Chan Two",
|
||||
"channelId": "chan002"},
|
||||
})
|
||||
rss_xml = _mock_rss_response([("vid001", "First Video"), ("vid002", "Second Video")])
|
||||
mock_response = MagicMock(status_code=200, text=rss_xml)
|
||||
|
||||
mock_process = MockProcess(stdout_data=stdout_bytes, returncode=0)
|
||||
|
||||
with patch("asyncio.create_subprocess_exec", return_value=mock_process):
|
||||
with patch("fetcher.build", return_value=mock_svc), \
|
||||
patch("fetcher.get_credentials", return_value="mock_creds"), \
|
||||
patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)):
|
||||
count = await fetcher.fetch_subscriptions()
|
||||
assert count == 1 # Only 1 new video should be added (vid456)
|
||||
assert count == 1 # Only vid002 is new
|
||||
|
||||
# Verify DB — vid001 still has original title (not updated)
|
||||
cursor = await test_db.execute("SELECT title FROM videos WHERE video_id='vid001'")
|
||||
row = await cursor.fetchone()
|
||||
assert row["title"] == "Old"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rotation_advances(test_db):
|
||||
"""Rotation slice should advance after each sync."""
|
||||
# Mock 150 channels
|
||||
sub_channels = [{"snippet": {"resourceId": {"channelId": f"chan{i:03d}"}}} for i in range(150)]
|
||||
mock_svc = _mock_youtube_build(sub_channels=sub_channels)
|
||||
|
||||
rss_xml = _mock_rss_response([]) # No videos, just test rotation
|
||||
mock_response = MagicMock(status_code=200, text=rss_xml)
|
||||
|
||||
with patch("fetcher.build", return_value=mock_svc), \
|
||||
patch("fetcher.get_credentials", return_value="mock_creds"), \
|
||||
patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)):
|
||||
# First sync — should process chan000-chan049
|
||||
count = await fetcher.fetch_subscriptions()
|
||||
# Second sync — chan050-chan099
|
||||
count2 = await fetcher.fetch_subscriptions()
|
||||
# Third sync — chan100-chan149
|
||||
count3 = await fetcher.fetch_subscriptions()
|
||||
# Fourth sync — wrap around, chan000-chan049 again
|
||||
count4 = await fetcher.fetch_subscriptions()
|
||||
|
||||
# Verify rotation file
|
||||
with open(fetcher._ROTATION_FILE) as f:
|
||||
state = json.load(f)
|
||||
assert state["offset"] == 50 # After 4th sync, offset = 50 (wrapped from 150)
|
||||
assert state["total"] == 150
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_quietly_handles_missing_credentials(test_db):
|
||||
"""When OAuth is missing, should return 0 gracefully."""
|
||||
with patch("fetcher.get_credentials", side_effect=FileNotFoundError("No client_secret")):
|
||||
count = await fetcher.fetch_subscriptions()
|
||||
assert count == 0
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
"""Tests for summarizer.py — mocks transcript API + DeepSeek API."""
|
||||
import pytest
|
||||
import os
|
||||
import httpx
|
||||
@@ -5,6 +6,7 @@ import aiosqlite
|
||||
import summarizer
|
||||
from unittest.mock import patch, MagicMock, AsyncMock
|
||||
|
||||
|
||||
class MockTranscript:
|
||||
def fetch(self):
|
||||
return [
|
||||
@@ -25,6 +27,17 @@ class MockTranscriptList:
|
||||
def __iter__(self):
|
||||
return iter([self.transcript])
|
||||
|
||||
|
||||
def _mock_deepseek_response(content: str) -> MagicMock:
|
||||
"""Build a DeepSeek/OpenAI-compatible API response."""
|
||||
r = MagicMock(spec=httpx.Response)
|
||||
r.status_code = 200
|
||||
r.json.return_value = {
|
||||
"choices": [{"message": {"content": content}}]
|
||||
}
|
||||
return r
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_transcript_success():
|
||||
mock_trans = MockTranscript()
|
||||
@@ -34,9 +47,9 @@ async def test_download_transcript_success():
|
||||
text = summarizer.download_transcript("test_vid")
|
||||
assert text == "Welcome to this video tutorial. Today we are learning about Svelte 5 and runes."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarize_video_success(test_db):
|
||||
# Insert a pending video in db first
|
||||
await test_db.execute("""
|
||||
INSERT INTO videos (video_id, title, url, status)
|
||||
VALUES ('summarize_vid', 'Video to Summarize', 'https://www.youtube.com/watch?v=summarize_vid', 'pending')
|
||||
@@ -45,19 +58,8 @@ async def test_summarize_video_success(test_db):
|
||||
|
||||
mock_trans = MockTranscript()
|
||||
mock_list = MockTranscriptList(mock_trans)
|
||||
mock_response = _mock_deepseek_response("**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line.")
|
||||
|
||||
# Mock Claude HTTP response
|
||||
mock_response = MagicMock(spec=httpx.Response)
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"content": [
|
||||
{
|
||||
"text": "**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
# Patch both transcript api and httpx AsyncClient post call
|
||||
with patch("youtube_transcript_api.YouTubeTranscriptApi.list", return_value=mock_list), \
|
||||
patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response) as mock_post:
|
||||
|
||||
@@ -69,12 +71,12 @@ async def test_summarize_video_success(test_db):
|
||||
assert summarizer.BASE_URL in url
|
||||
|
||||
headers = mock_post.call_args[1]["headers"]
|
||||
assert "x-api-key" in headers
|
||||
assert "Authorization" in headers
|
||||
assert headers["User-Agent"] is not None
|
||||
|
||||
payload = mock_post.call_args[1]["json"]
|
||||
assert payload["model"] == "claude-haiku-4-5-20251001"
|
||||
assert len(payload["messages"]) == 1
|
||||
assert payload["model"] == "deepseek-chat"
|
||||
assert len(payload["messages"]) == 2 # system + user
|
||||
|
||||
assert summary == "**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line."
|
||||
|
||||
@@ -85,9 +87,40 @@ async def test_summarize_video_success(test_db):
|
||||
assert video["transcript"] == "Welcome to this video tutorial. Today we are learning about Svelte 5 and runes."
|
||||
assert video["summary"] == "**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarize_video_deepseek_error_leaves_null(test_db):
|
||||
"""When DeepSeek fails, summary should stay NULL for retry."""
|
||||
await test_db.execute("""
|
||||
INSERT INTO videos (video_id, title, url, status)
|
||||
VALUES ('err_vid', 'Error Video', 'https://youtube.com/watch?v=err_vid', 'pending')
|
||||
""")
|
||||
await test_db.commit()
|
||||
|
||||
mock_trans = MockTranscript()
|
||||
mock_list = MockTranscriptList(mock_trans)
|
||||
mock_response = MagicMock(spec=httpx.Response)
|
||||
mock_response.status_code = 429
|
||||
# raise_for_status on 429 should raise HTTPStatusError
|
||||
mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
|
||||
"429 Rate Limited", request=MagicMock(), response=mock_response
|
||||
)
|
||||
|
||||
with patch("youtube_transcript_api.YouTubeTranscriptApi.list", return_value=mock_list), \
|
||||
patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response):
|
||||
|
||||
result = await summarizer.summarize_video("err_vid")
|
||||
|
||||
assert result is None # Returns None instead of an error string
|
||||
|
||||
# Verify DB — summary should remain NULL
|
||||
cursor = await test_db.execute("SELECT summary FROM videos WHERE video_id = 'err_vid'")
|
||||
row = await cursor.fetchone()
|
||||
assert row["summary"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarize_all_pending(test_db):
|
||||
# Insert multiple videos, one pending and one already summarized
|
||||
await test_db.execute("""
|
||||
INSERT INTO videos (video_id, title, url, summary, status)
|
||||
VALUES
|
||||
@@ -98,28 +131,21 @@ async def test_summarize_all_pending(test_db):
|
||||
|
||||
mock_trans = MockTranscript()
|
||||
mock_list = MockTranscriptList(mock_trans)
|
||||
|
||||
mock_response = MagicMock(spec=httpx.Response)
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"content": [{"text": "Fresh summary"}]
|
||||
}
|
||||
mock_response = _mock_deepseek_response("Fresh summary")
|
||||
|
||||
with patch("youtube_transcript_api.YouTubeTranscriptApi.list", return_value=mock_list), \
|
||||
patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response) as mock_post:
|
||||
|
||||
await summarizer.summarize_all_pending()
|
||||
|
||||
# Should only call API once for 'pending_vid'
|
||||
# Should only call API once for 'pending_vid' (done_vid already has summary)
|
||||
mock_post.assert_called_once()
|
||||
payload = mock_post.call_args[1]["json"]
|
||||
assert "Welcome to this video tutorial." in payload["messages"][0]["content"]
|
||||
|
||||
# Verify db states
|
||||
cursor = await test_db.execute("SELECT video_id, summary FROM videos ORDER BY video_id")
|
||||
videos = await cursor.fetchall()
|
||||
assert videos[0]["video_id"] == "done_vid"
|
||||
assert videos[0]["summary"] == "Existing summary" # Untouched
|
||||
assert videos[0]["summary"] == "Existing summary" # Untouched
|
||||
|
||||
assert videos[1]["video_id"] == "pending_vid"
|
||||
assert videos[1]["summary"] == "Fresh summary" # Updated
|
||||
assert videos[1]["summary"] == "Fresh summary" # Updated
|
||||
|
||||
Reference in New Issue
Block a user