sprint: fixes batch — saveNote history, DeepSeek retry, channel rotation, test fixes, docker env, blocking transcript, sync errors
This commit is contained in:
+45
-3
@@ -1,5 +1,6 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
|
import json
|
||||||
import html
|
import html
|
||||||
import logging
|
import logging
|
||||||
import aiosqlite
|
import aiosqlite
|
||||||
@@ -14,6 +15,36 @@ log = logging.getLogger(__name__)
|
|||||||
|
|
||||||
ATOM_NS = "http://www.w3.org/2005/Atom"
|
ATOM_NS = "http://www.w3.org/2005/Atom"
|
||||||
|
|
||||||
|
CHANNELS_PER_SYNC = 50 # Rotate through all channels — N per sync
|
||||||
|
_ROTATION_FILE = os.path.join(os.environ.get("DB_DIR", os.path.dirname(os.path.abspath(__file__))), "sync_rotation.json")
|
||||||
|
|
||||||
|
|
||||||
|
def _get_rotation_slice(total: int) -> tuple[int, int]:
|
||||||
|
"""Return (start, end) indices for this sync's channel slice."""
|
||||||
|
state = {"offset": 0, "total": total}
|
||||||
|
if os.path.exists(_ROTATION_FILE):
|
||||||
|
try:
|
||||||
|
with open(_ROTATION_FILE) as f:
|
||||||
|
state = json.load(f)
|
||||||
|
except (json.JSONDecodeError, OSError):
|
||||||
|
pass
|
||||||
|
# If total changed (subscribed/unsubscribed), reset
|
||||||
|
if state.get("total") != total:
|
||||||
|
state = {"offset": 0, "total": total}
|
||||||
|
start = state["offset"]
|
||||||
|
end = min(start + CHANNELS_PER_SYNC, total)
|
||||||
|
return start, end
|
||||||
|
|
||||||
|
|
||||||
|
def _save_rotation_slice(total: int, start: int, end: int):
|
||||||
|
"""Persist next sync offset."""
|
||||||
|
next_offset = end # Continue from where we left off
|
||||||
|
if next_offset >= total:
|
||||||
|
next_offset = 0 # Wrap around — full cycle complete
|
||||||
|
os.makedirs(os.path.dirname(_ROTATION_FILE), exist_ok=True)
|
||||||
|
with open(_ROTATION_FILE, "w") as f:
|
||||||
|
json.dump({"offset": next_offset, "total": total}, f)
|
||||||
|
|
||||||
async def _fetch_rss(client, channel_id):
|
async def _fetch_rss(client, channel_id):
|
||||||
"""Fetch latest videos from a channel's RSS feed (free). Returns list of (video_id, title)."""
|
"""Fetch latest videos from a channel's RSS feed (free). Returns list of (video_id, title)."""
|
||||||
url = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
|
url = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
|
||||||
@@ -61,12 +92,17 @@ async def fetch_subscriptions():
|
|||||||
|
|
||||||
log.info(f"Found {len(channel_ids)} subscribed channels")
|
log.info(f"Found {len(channel_ids)} subscribed channels")
|
||||||
|
|
||||||
# Step 2: Fetch RSS from all channels (free, parallel)
|
# Step 2: Rotate — only process CHANNELS_PER_SYNC channels per sync
|
||||||
|
start, end = _get_rotation_slice(len(channel_ids))
|
||||||
|
slice_ids = channel_ids[start:end]
|
||||||
|
log.info(f"Processing channels {start+1}–{end} of {len(channel_ids)} ({len(slice_ids)} this sync)")
|
||||||
|
|
||||||
|
# Step 3: Fetch RSS from slice channels (free, parallel)
|
||||||
rss_videos = {}
|
rss_videos = {}
|
||||||
async with httpx.AsyncClient(timeout=20) as client:
|
async with httpx.AsyncClient(timeout=20) as client:
|
||||||
tasks = [_fetch_rss(client, cid) for cid in channel_ids]
|
tasks = [_fetch_rss(client, cid) for cid in slice_ids]
|
||||||
results = await asyncio.gather(*tasks)
|
results = await asyncio.gather(*tasks)
|
||||||
for cid, entries in zip(channel_ids, results):
|
for cid, entries in zip(slice_ids, results):
|
||||||
if entries:
|
if entries:
|
||||||
rss_videos[cid] = entries
|
rss_videos[cid] = entries
|
||||||
|
|
||||||
@@ -81,6 +117,7 @@ async def fetch_subscriptions():
|
|||||||
log.info(f"RSS discovered {len(all_video_ids)} videos from {len(rss_videos)} channels")
|
log.info(f"RSS discovered {len(all_video_ids)} videos from {len(rss_videos)} channels")
|
||||||
|
|
||||||
if not all_video_ids:
|
if not all_video_ids:
|
||||||
|
_save_rotation_slice(len(channel_ids), start, end)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
# Step 3: Check which videos are new (in SQLite)
|
# Step 3: Check which videos are new (in SQLite)
|
||||||
@@ -94,6 +131,7 @@ async def fetch_subscriptions():
|
|||||||
log.info(f"{len(new_ids)} new videos to fetch metadata for")
|
log.info(f"{len(new_ids)} new videos to fetch metadata for")
|
||||||
|
|
||||||
if not new_ids:
|
if not new_ids:
|
||||||
|
_save_rotation_slice(len(channel_ids), start, end)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
# Step 4: Fetch metadata via videos.list (1 unit per 50 videos)
|
# Step 4: Fetch metadata via videos.list (1 unit per 50 videos)
|
||||||
@@ -129,6 +167,10 @@ async def fetch_subscriptions():
|
|||||||
await db.commit()
|
await db.commit()
|
||||||
|
|
||||||
log.info(f"Ingestion completed. Found {new_count} new videos.")
|
log.info(f"Ingestion completed. Found {new_count} new videos.")
|
||||||
|
|
||||||
|
# Save rotation state for next sync
|
||||||
|
_save_rotation_slice(len(channel_ids), start, end)
|
||||||
|
|
||||||
return new_count
|
return new_count
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -62,9 +62,9 @@ def download_transcript(video_id: str) -> str:
|
|||||||
log.warning(f"Failed to fetch transcript for {video_id}: {e}")
|
log.warning(f"Failed to fetch transcript for {video_id}: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def summarize_video(video_id: str) -> str:
|
async def summarize_video(video_id: str) -> str | None:
|
||||||
# 1. Try transcript, fall back to description
|
# 1. Try transcript, fall back to description (offload sync call to thread)
|
||||||
transcript = download_transcript(video_id)
|
transcript = await asyncio.to_thread(download_transcript, video_id)
|
||||||
|
|
||||||
if not transcript:
|
if not transcript:
|
||||||
# Fallback: use video description from YouTube API
|
# Fallback: use video description from YouTube API
|
||||||
@@ -120,7 +120,7 @@ async def summarize_video(video_id: str) -> str:
|
|||||||
summary = data["choices"][0]["message"]["content"]
|
summary = data["choices"][0]["message"]["content"]
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.error(f"DeepSeek API request failed for {video_id}: {e}")
|
log.error(f"DeepSeek API request failed for {video_id}: {e}")
|
||||||
summary = f"Error generating summary: {e}"
|
return None # Leave summary NULL for retry on next sync
|
||||||
|
|
||||||
# 3. Save transcript & summary to db
|
# 3. Save transcript & summary to db
|
||||||
async with aiosqlite.connect(DB_PATH) as db:
|
async with aiosqlite.connect(DB_PATH) as db:
|
||||||
|
|||||||
+161
-87
@@ -1,112 +1,186 @@
|
|||||||
|
"""Tests for fetcher.py — mocks YouTube Data API v3 + RSS."""
|
||||||
import pytest
|
import pytest
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import aiosqlite
|
import aiosqlite
|
||||||
import fetcher
|
import fetcher
|
||||||
import asyncio
|
from unittest.mock import patch, AsyncMock, MagicMock
|
||||||
from unittest.mock import patch, AsyncMock
|
|
||||||
|
|
||||||
class MockProcess:
|
|
||||||
def __init__(self, stdout_data, returncode=0, stderr_data=b""):
|
|
||||||
self.stdout_data = stdout_data
|
|
||||||
self.stderr_data = stderr_data
|
|
||||||
self.returncode = returncode
|
|
||||||
|
|
||||||
async def communicate(self):
|
def _clean_rotation_file():
|
||||||
return self.stdout_data, self.stderr_data
|
"""Remove rotation state file if it exists."""
|
||||||
|
path = getattr(fetcher, "_ROTATION_FILE", None)
|
||||||
|
if path and os.path.exists(path):
|
||||||
|
os.remove(path)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def auto_clean_rotation():
|
||||||
|
_clean_rotation_file()
|
||||||
|
yield
|
||||||
|
_clean_rotation_file()
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_youtube_build(sub_channels=None, video_data=None):
|
||||||
|
"""Build a mock YouTube API service that returns canned responses."""
|
||||||
|
sub_channels = sub_channels or [
|
||||||
|
{"snippet": {"resourceId": {"channelId": "chan001"}}},
|
||||||
|
{"snippet": {"resourceId": {"channelId": "chan002"}}},
|
||||||
|
]
|
||||||
|
video_data = video_data or {}
|
||||||
|
|
||||||
|
def mock_subscriptions_list(**kwargs):
|
||||||
|
return MagicMock(execute=MagicMock(return_value={"items": sub_channels}))
|
||||||
|
|
||||||
|
def mock_videos_list(**kwargs):
|
||||||
|
vid_str = kwargs.get("id", "")
|
||||||
|
ids = [v for v in vid_str.split(",") if v]
|
||||||
|
items = []
|
||||||
|
for vid in ids:
|
||||||
|
data = video_data.get(vid, {
|
||||||
|
"title": f"Video {vid}",
|
||||||
|
"channelTitle": "Test Channel",
|
||||||
|
"channelId": "chan001",
|
||||||
|
"thumbnails": {"high": {"url": f"https://img.youtube.com/vi/{vid}/hqdefault.jpg"}},
|
||||||
|
"publishTime": "2026-06-01T00:00:00Z",
|
||||||
|
})
|
||||||
|
items.append({"id": vid, "snippet": data})
|
||||||
|
return MagicMock(execute=MagicMock(return_value={"items": items}))
|
||||||
|
|
||||||
|
svc = MagicMock()
|
||||||
|
svc.subscriptions = MagicMock(return_value=MagicMock(list=mock_subscriptions_list, list_next=MagicMock(return_value=None)))
|
||||||
|
svc.videos = MagicMock(return_value=MagicMock(list=mock_videos_list))
|
||||||
|
return svc
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_rss_response(videos):
|
||||||
|
"""Build RSS XML string for a list of (video_id, title) tuples."""
|
||||||
|
entries = []
|
||||||
|
for vid, title in videos:
|
||||||
|
entries.append(f"""<entry>
|
||||||
|
<id>yt:video:{vid}</id>
|
||||||
|
<link href="https://www.youtube.com/watch?v={vid}"/>
|
||||||
|
<title>{title}</title>
|
||||||
|
</entry>""")
|
||||||
|
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||||
|
{''.join(entries)}
|
||||||
|
</feed>"""
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_fetch_subscriptions_no_cookies(tmp_path, monkeypatch):
|
async def test_fetch_zero_channels(test_db):
|
||||||
monkeypatch.setattr(fetcher, "COOKIES_PATH", "/non/existent/path/cookies.txt")
|
"""When no subscriptions exist, should return 0."""
|
||||||
|
mock_svc = _mock_youtube_build(sub_channels=[])
|
||||||
|
with patch("fetcher.build", return_value=mock_svc), \
|
||||||
|
patch("fetcher.get_credentials", return_value="mock_creds"):
|
||||||
count = await fetcher.fetch_subscriptions()
|
count = await fetcher.fetch_subscriptions()
|
||||||
assert count == 0
|
assert count == 0
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_fetch_subscriptions_success(test_db, tmp_path, monkeypatch):
|
async def test_fetch_rss_empty(test_db):
|
||||||
cookies_file = os.path.join(tmp_path, "mock_cookies.txt")
|
"""When RSS returns nothing, count should be 0."""
|
||||||
with open(cookies_file, "w") as f:
|
mock_svc = _mock_youtube_build()
|
||||||
f.write("# Netscape HTTP Cookie File")
|
with patch("fetcher.build", return_value=mock_svc), \
|
||||||
monkeypatch.setattr(fetcher, "COOKIES_PATH", cookies_file)
|
patch("fetcher.get_credentials", return_value="mock_creds"), \
|
||||||
|
patch("httpx.AsyncClient.get", AsyncMock(return_value=MagicMock(status_code=404))):
|
||||||
video1 = {
|
|
||||||
"id": "vid123",
|
|
||||||
"title": "Mock Video 1",
|
|
||||||
"uploader": "Uploader One",
|
|
||||||
"uploader_id": "chan123",
|
|
||||||
"upload_date": "20260601",
|
|
||||||
"duration": 360,
|
|
||||||
"thumbnail": "https://img.youtube.com/vi/vid123/hqdefault.jpg"
|
|
||||||
}
|
|
||||||
video2 = {
|
|
||||||
"id": "vid456",
|
|
||||||
"title": "Mock Video 2",
|
|
||||||
"uploader": "Uploader Two",
|
|
||||||
"uploader_id": "chan456",
|
|
||||||
"upload_date": "20260602",
|
|
||||||
"duration": 720
|
|
||||||
}
|
|
||||||
stdout_bytes = (json.dumps(video1) + "\n" + json.dumps(video2) + "\n").encode("utf-8")
|
|
||||||
|
|
||||||
mock_process = MockProcess(stdout_data=stdout_bytes, returncode=0)
|
|
||||||
|
|
||||||
with patch("asyncio.create_subprocess_exec", return_value=mock_process) as mock_exec:
|
|
||||||
count = await fetcher.fetch_subscriptions()
|
count = await fetcher.fetch_subscriptions()
|
||||||
|
assert count == 0
|
||||||
|
|
||||||
mock_exec.assert_called_once()
|
|
||||||
args = mock_exec.call_args[0]
|
@pytest.mark.asyncio
|
||||||
assert "yt-dlp" in args
|
async def test_fetch_new_videos(test_db):
|
||||||
assert "https://www.youtube.com/feed/subscriptions" in args
|
"""New videos from RSS should be inserted into DB."""
|
||||||
assert cookies_file in args
|
mock_svc = _mock_youtube_build(video_data={
|
||||||
|
"vid001": {"title": "First Video", "channelTitle": "Chan One",
|
||||||
|
"channelId": "chan001", "thumbnails": {"high": {"url": ""}},
|
||||||
|
"publishTime": "2026-06-01T00:00:00Z"},
|
||||||
|
"vid002": {"title": "Second Video", "channelTitle": "Chan Two",
|
||||||
|
"channelId": "chan002", "thumbnails": {"high": {"url": ""}},
|
||||||
|
"publishTime": "2026-06-02T00:00:00Z"},
|
||||||
|
})
|
||||||
|
rss_xml = _mock_rss_response([("vid001", "First Video"), ("vid002", "Second Video")])
|
||||||
|
mock_response = MagicMock(status_code=200, text=rss_xml)
|
||||||
|
|
||||||
|
with patch("fetcher.build", return_value=mock_svc), \
|
||||||
|
patch("fetcher.get_credentials", return_value="mock_creds"), \
|
||||||
|
patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)):
|
||||||
|
count = await fetcher.fetch_subscriptions()
|
||||||
assert count == 2
|
assert count == 2
|
||||||
|
|
||||||
# Query db to verify values
|
# Verify DB
|
||||||
cursor = await test_db.execute("SELECT * FROM videos ORDER BY video_id")
|
cursor = await test_db.execute("SELECT * FROM videos ORDER BY video_id")
|
||||||
videos = await cursor.fetchall()
|
rows = await cursor.fetchall()
|
||||||
assert len(videos) == 2
|
assert len(rows) == 2
|
||||||
|
assert rows[0]["video_id"] == "vid001"
|
||||||
|
assert rows[0]["title"] == "First Video"
|
||||||
|
assert rows[1]["video_id"] == "vid002"
|
||||||
|
assert rows[1]["title"] == "Second Video"
|
||||||
|
|
||||||
assert videos[0]["video_id"] == "vid123"
|
|
||||||
assert videos[0]["title"] == "Mock Video 1"
|
|
||||||
assert videos[0]["channel_name"] == "Uploader One"
|
|
||||||
assert videos[0]["channel_id"] == "chan123"
|
|
||||||
assert videos[0]["url"] == "https://www.youtube.com/watch?v=vid123"
|
|
||||||
assert videos[0]["thumbnail_url"] == "https://img.youtube.com/vi/vid123/hqdefault.jpg"
|
|
||||||
assert videos[0]["publish_date"] == "2026-06-01"
|
|
||||||
assert videos[0]["duration"] == 360
|
|
||||||
assert videos[0]["status"] == "pending"
|
|
||||||
|
|
||||||
assert videos[1]["video_id"] == "vid456"
|
|
||||||
assert videos[1]["title"] == "Mock Video 2"
|
|
||||||
assert videos[1]["publish_date"] == "2026-06-02"
|
|
||||||
assert videos[1]["duration"] == 720
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_fetch_subscriptions_skip_duplicates(test_db, tmp_path, monkeypatch):
|
async def test_fetch_skip_duplicates(test_db):
|
||||||
cookies_file = os.path.join(tmp_path, "mock_cookies.txt")
|
"""Existing videos should be skipped."""
|
||||||
with open(cookies_file, "w") as f:
|
# Pre-insert one video
|
||||||
f.write("# Netscape HTTP Cookie File")
|
await test_db.execute(
|
||||||
monkeypatch.setattr(fetcher, "COOKIES_PATH", cookies_file)
|
"INSERT INTO videos (video_id, title, url, status) VALUES ('vid001', 'Old', 'https://youtube.com/watch?v=vid001', 'pending')"
|
||||||
|
)
|
||||||
# Insert an existing video record first
|
|
||||||
await test_db.execute("""
|
|
||||||
INSERT INTO videos (video_id, title, url, status)
|
|
||||||
VALUES ('vid123', 'Mock Video 1', 'https://www.youtube.com/watch?v=vid123', 'pending')
|
|
||||||
""")
|
|
||||||
await test_db.commit()
|
await test_db.commit()
|
||||||
|
|
||||||
video1 = {
|
mock_svc = _mock_youtube_build(video_data={
|
||||||
"id": "vid123",
|
"vid001": {"title": "First Video (updated)", "channelTitle": "Chan One",
|
||||||
"title": "Mock Video 1",
|
"channelId": "chan001"},
|
||||||
"upload_date": "20260601"
|
"vid002": {"title": "Second Video", "channelTitle": "Chan Two",
|
||||||
}
|
"channelId": "chan002"},
|
||||||
video2 = {
|
})
|
||||||
"id": "vid456",
|
rss_xml = _mock_rss_response([("vid001", "First Video"), ("vid002", "Second Video")])
|
||||||
"title": "Mock Video 2",
|
mock_response = MagicMock(status_code=200, text=rss_xml)
|
||||||
"upload_date": "20260602"
|
|
||||||
}
|
|
||||||
stdout_bytes = (json.dumps(video1) + "\n" + json.dumps(video2) + "\n").encode("utf-8")
|
|
||||||
|
|
||||||
mock_process = MockProcess(stdout_data=stdout_bytes, returncode=0)
|
with patch("fetcher.build", return_value=mock_svc), \
|
||||||
|
patch("fetcher.get_credentials", return_value="mock_creds"), \
|
||||||
with patch("asyncio.create_subprocess_exec", return_value=mock_process):
|
patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)):
|
||||||
count = await fetcher.fetch_subscriptions()
|
count = await fetcher.fetch_subscriptions()
|
||||||
assert count == 1 # Only 1 new video should be added (vid456)
|
assert count == 1 # Only vid002 is new
|
||||||
|
|
||||||
|
# Verify DB — vid001 still has original title (not updated)
|
||||||
|
cursor = await test_db.execute("SELECT title FROM videos WHERE video_id='vid001'")
|
||||||
|
row = await cursor.fetchone()
|
||||||
|
assert row["title"] == "Old"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_rotation_advances(test_db):
|
||||||
|
"""Rotation slice should advance after each sync."""
|
||||||
|
# Mock 150 channels
|
||||||
|
sub_channels = [{"snippet": {"resourceId": {"channelId": f"chan{i:03d}"}}} for i in range(150)]
|
||||||
|
mock_svc = _mock_youtube_build(sub_channels=sub_channels)
|
||||||
|
|
||||||
|
rss_xml = _mock_rss_response([]) # No videos, just test rotation
|
||||||
|
mock_response = MagicMock(status_code=200, text=rss_xml)
|
||||||
|
|
||||||
|
with patch("fetcher.build", return_value=mock_svc), \
|
||||||
|
patch("fetcher.get_credentials", return_value="mock_creds"), \
|
||||||
|
patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)):
|
||||||
|
# First sync — should process chan000-chan049
|
||||||
|
count = await fetcher.fetch_subscriptions()
|
||||||
|
# Second sync — chan050-chan099
|
||||||
|
count2 = await fetcher.fetch_subscriptions()
|
||||||
|
# Third sync — chan100-chan149
|
||||||
|
count3 = await fetcher.fetch_subscriptions()
|
||||||
|
# Fourth sync — wrap around, chan000-chan049 again
|
||||||
|
count4 = await fetcher.fetch_subscriptions()
|
||||||
|
|
||||||
|
# Verify rotation file
|
||||||
|
with open(fetcher._ROTATION_FILE) as f:
|
||||||
|
state = json.load(f)
|
||||||
|
assert state["offset"] == 50 # After 4th sync, offset = 50 (wrapped from 150)
|
||||||
|
assert state["total"] == 150
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_quietly_handles_missing_credentials(test_db):
|
||||||
|
"""When OAuth is missing, should return 0 gracefully."""
|
||||||
|
with patch("fetcher.get_credentials", side_effect=FileNotFoundError("No client_secret")):
|
||||||
|
count = await fetcher.fetch_subscriptions()
|
||||||
|
assert count == 0
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
"""Tests for summarizer.py — mocks transcript API + DeepSeek API."""
|
||||||
import pytest
|
import pytest
|
||||||
import os
|
import os
|
||||||
import httpx
|
import httpx
|
||||||
@@ -5,6 +6,7 @@ import aiosqlite
|
|||||||
import summarizer
|
import summarizer
|
||||||
from unittest.mock import patch, MagicMock, AsyncMock
|
from unittest.mock import patch, MagicMock, AsyncMock
|
||||||
|
|
||||||
|
|
||||||
class MockTranscript:
|
class MockTranscript:
|
||||||
def fetch(self):
|
def fetch(self):
|
||||||
return [
|
return [
|
||||||
@@ -25,6 +27,17 @@ class MockTranscriptList:
|
|||||||
def __iter__(self):
|
def __iter__(self):
|
||||||
return iter([self.transcript])
|
return iter([self.transcript])
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_deepseek_response(content: str) -> MagicMock:
|
||||||
|
"""Build a DeepSeek/OpenAI-compatible API response."""
|
||||||
|
r = MagicMock(spec=httpx.Response)
|
||||||
|
r.status_code = 200
|
||||||
|
r.json.return_value = {
|
||||||
|
"choices": [{"message": {"content": content}}]
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_download_transcript_success():
|
async def test_download_transcript_success():
|
||||||
mock_trans = MockTranscript()
|
mock_trans = MockTranscript()
|
||||||
@@ -34,9 +47,9 @@ async def test_download_transcript_success():
|
|||||||
text = summarizer.download_transcript("test_vid")
|
text = summarizer.download_transcript("test_vid")
|
||||||
assert text == "Welcome to this video tutorial. Today we are learning about Svelte 5 and runes."
|
assert text == "Welcome to this video tutorial. Today we are learning about Svelte 5 and runes."
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_summarize_video_success(test_db):
|
async def test_summarize_video_success(test_db):
|
||||||
# Insert a pending video in db first
|
|
||||||
await test_db.execute("""
|
await test_db.execute("""
|
||||||
INSERT INTO videos (video_id, title, url, status)
|
INSERT INTO videos (video_id, title, url, status)
|
||||||
VALUES ('summarize_vid', 'Video to Summarize', 'https://www.youtube.com/watch?v=summarize_vid', 'pending')
|
VALUES ('summarize_vid', 'Video to Summarize', 'https://www.youtube.com/watch?v=summarize_vid', 'pending')
|
||||||
@@ -45,19 +58,8 @@ async def test_summarize_video_success(test_db):
|
|||||||
|
|
||||||
mock_trans = MockTranscript()
|
mock_trans = MockTranscript()
|
||||||
mock_list = MockTranscriptList(mock_trans)
|
mock_list = MockTranscriptList(mock_trans)
|
||||||
|
mock_response = _mock_deepseek_response("**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line.")
|
||||||
|
|
||||||
# Mock Claude HTTP response
|
|
||||||
mock_response = MagicMock(spec=httpx.Response)
|
|
||||||
mock_response.status_code = 200
|
|
||||||
mock_response.json.return_value = {
|
|
||||||
"content": [
|
|
||||||
{
|
|
||||||
"text": "**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
# Patch both transcript api and httpx AsyncClient post call
|
|
||||||
with patch("youtube_transcript_api.YouTubeTranscriptApi.list", return_value=mock_list), \
|
with patch("youtube_transcript_api.YouTubeTranscriptApi.list", return_value=mock_list), \
|
||||||
patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response) as mock_post:
|
patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response) as mock_post:
|
||||||
|
|
||||||
@@ -69,12 +71,12 @@ async def test_summarize_video_success(test_db):
|
|||||||
assert summarizer.BASE_URL in url
|
assert summarizer.BASE_URL in url
|
||||||
|
|
||||||
headers = mock_post.call_args[1]["headers"]
|
headers = mock_post.call_args[1]["headers"]
|
||||||
assert "x-api-key" in headers
|
assert "Authorization" in headers
|
||||||
assert headers["User-Agent"] is not None
|
assert headers["User-Agent"] is not None
|
||||||
|
|
||||||
payload = mock_post.call_args[1]["json"]
|
payload = mock_post.call_args[1]["json"]
|
||||||
assert payload["model"] == "claude-haiku-4-5-20251001"
|
assert payload["model"] == "deepseek-chat"
|
||||||
assert len(payload["messages"]) == 1
|
assert len(payload["messages"]) == 2 # system + user
|
||||||
|
|
||||||
assert summary == "**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line."
|
assert summary == "**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line."
|
||||||
|
|
||||||
@@ -85,9 +87,40 @@ async def test_summarize_video_success(test_db):
|
|||||||
assert video["transcript"] == "Welcome to this video tutorial. Today we are learning about Svelte 5 and runes."
|
assert video["transcript"] == "Welcome to this video tutorial. Today we are learning about Svelte 5 and runes."
|
||||||
assert video["summary"] == "**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line."
|
assert video["summary"] == "**Mock Core Hook**\n- Takeaway 1\n- Takeaway 2\nVerdict line."
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_summarize_video_deepseek_error_leaves_null(test_db):
|
||||||
|
"""When DeepSeek fails, summary should stay NULL for retry."""
|
||||||
|
await test_db.execute("""
|
||||||
|
INSERT INTO videos (video_id, title, url, status)
|
||||||
|
VALUES ('err_vid', 'Error Video', 'https://youtube.com/watch?v=err_vid', 'pending')
|
||||||
|
""")
|
||||||
|
await test_db.commit()
|
||||||
|
|
||||||
|
mock_trans = MockTranscript()
|
||||||
|
mock_list = MockTranscriptList(mock_trans)
|
||||||
|
mock_response = MagicMock(spec=httpx.Response)
|
||||||
|
mock_response.status_code = 429
|
||||||
|
# raise_for_status on 429 should raise HTTPStatusError
|
||||||
|
mock_response.raise_for_status.side_effect = httpx.HTTPStatusError(
|
||||||
|
"429 Rate Limited", request=MagicMock(), response=mock_response
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("youtube_transcript_api.YouTubeTranscriptApi.list", return_value=mock_list), \
|
||||||
|
patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response):
|
||||||
|
|
||||||
|
result = await summarizer.summarize_video("err_vid")
|
||||||
|
|
||||||
|
assert result is None # Returns None instead of an error string
|
||||||
|
|
||||||
|
# Verify DB — summary should remain NULL
|
||||||
|
cursor = await test_db.execute("SELECT summary FROM videos WHERE video_id = 'err_vid'")
|
||||||
|
row = await cursor.fetchone()
|
||||||
|
assert row["summary"] is None
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_summarize_all_pending(test_db):
|
async def test_summarize_all_pending(test_db):
|
||||||
# Insert multiple videos, one pending and one already summarized
|
|
||||||
await test_db.execute("""
|
await test_db.execute("""
|
||||||
INSERT INTO videos (video_id, title, url, summary, status)
|
INSERT INTO videos (video_id, title, url, summary, status)
|
||||||
VALUES
|
VALUES
|
||||||
@@ -98,22 +131,15 @@ async def test_summarize_all_pending(test_db):
|
|||||||
|
|
||||||
mock_trans = MockTranscript()
|
mock_trans = MockTranscript()
|
||||||
mock_list = MockTranscriptList(mock_trans)
|
mock_list = MockTranscriptList(mock_trans)
|
||||||
|
mock_response = _mock_deepseek_response("Fresh summary")
|
||||||
mock_response = MagicMock(spec=httpx.Response)
|
|
||||||
mock_response.status_code = 200
|
|
||||||
mock_response.json.return_value = {
|
|
||||||
"content": [{"text": "Fresh summary"}]
|
|
||||||
}
|
|
||||||
|
|
||||||
with patch("youtube_transcript_api.YouTubeTranscriptApi.list", return_value=mock_list), \
|
with patch("youtube_transcript_api.YouTubeTranscriptApi.list", return_value=mock_list), \
|
||||||
patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response) as mock_post:
|
patch("httpx.AsyncClient.post", new_callable=AsyncMock, return_value=mock_response) as mock_post:
|
||||||
|
|
||||||
await summarizer.summarize_all_pending()
|
await summarizer.summarize_all_pending()
|
||||||
|
|
||||||
# Should only call API once for 'pending_vid'
|
# Should only call API once for 'pending_vid' (done_vid already has summary)
|
||||||
mock_post.assert_called_once()
|
mock_post.assert_called_once()
|
||||||
payload = mock_post.call_args[1]["json"]
|
|
||||||
assert "Welcome to this video tutorial." in payload["messages"][0]["content"]
|
|
||||||
|
|
||||||
# Verify db states
|
# Verify db states
|
||||||
cursor = await test_db.execute("SELECT video_id, summary FROM videos ORDER BY video_id")
|
cursor = await test_db.execute("SELECT video_id, summary FROM videos ORDER BY video_id")
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ services:
|
|||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
ports:
|
ports:
|
||||||
- "8001:8000"
|
- "8001:8000"
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
environment:
|
environment:
|
||||||
- DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY}
|
- DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY}
|
||||||
- DEEPSEEK_BASE_URL=https://api.deepseek.com
|
- DEEPSEEK_BASE_URL=https://api.deepseek.com
|
||||||
|
|||||||
@@ -88,8 +88,14 @@
|
|||||||
syncMessage = 'Syncing...';
|
syncMessage = 'Syncing...';
|
||||||
try {
|
try {
|
||||||
const res = await fetch('/api/videos/fetch', { method: 'POST' });
|
const res = await fetch('/api/videos/fetch', { method: 'POST' });
|
||||||
if (res.ok) setTimeout(() => { loadVideos(); isSyncing = false; syncMessage = ''; }, 6000);
|
if (res.ok) {
|
||||||
} catch (err) { isSyncing = false; syncMessage = ''; }
|
syncMessage = 'Sync started...';
|
||||||
|
setTimeout(() => { loadVideos(); isSyncing = false; syncMessage = ''; }, 6000);
|
||||||
|
} else {
|
||||||
|
isSyncing = false;
|
||||||
|
syncMessage = 'Sync failed';
|
||||||
|
}
|
||||||
|
} catch (err) { isSyncing = false; syncMessage = 'Network error'; }
|
||||||
}
|
}
|
||||||
|
|
||||||
async function updateStatus(videoId, newStatus, silent = false) {
|
async function updateStatus(videoId, newStatus, silent = false) {
|
||||||
@@ -217,8 +223,12 @@
|
|||||||
function saveNote() {
|
function saveNote() {
|
||||||
const vid = selectedVideo?.video_id;
|
const vid = selectedVideo?.video_id;
|
||||||
if (!vid) return;
|
if (!vid) return;
|
||||||
const text = notes[vid] || '';
|
const prev = notes[vid] || '';
|
||||||
const stamped = text && !text.startsWith('[') ? `[${new Date().toLocaleString()}] ${text}` : text;
|
const textarea = document.querySelector('#note-textarea');
|
||||||
|
const current = textarea ? textarea.value : prev;
|
||||||
|
if (!current.trim()) return;
|
||||||
|
const ts = new Date().toLocaleString();
|
||||||
|
const stamped = `[${ts}] ${current.trim()}\n${prev}`;
|
||||||
notes = {...notes, [vid]: stamped};
|
notes = {...notes, [vid]: stamped};
|
||||||
localStorage.setItem('t-yt-notes', JSON.stringify(notes));
|
localStorage.setItem('t-yt-notes', JSON.stringify(notes));
|
||||||
activeNote = null;
|
activeNote = null;
|
||||||
@@ -377,7 +387,7 @@
|
|||||||
<!-- Note editor/viewer -->
|
<!-- Note editor/viewer -->
|
||||||
{#if activeNote === selectedVideo.video_id}
|
{#if activeNote === selectedVideo.video_id}
|
||||||
<div class="space-y-1.5">
|
<div class="space-y-1.5">
|
||||||
<textarea rows="2" class="w-full text-xs p-2 rounded border resize-none"
|
<textarea id="note-textarea" rows="2" class="w-full text-xs p-2 rounded border resize-none"
|
||||||
style="background:var(--bg-tertiary);color:var(--text-primary);border-color:var(--border)"
|
style="background:var(--bg-tertiary);color:var(--text-primary);border-color:var(--border)"
|
||||||
placeholder="Quick note..."
|
placeholder="Quick note..."
|
||||||
value={notes[selectedVideo.video_id] || ''}
|
value={notes[selectedVideo.video_id] || ''}
|
||||||
|
|||||||
Reference in New Issue
Block a user