diff --git a/dashboard/backend/config.py b/dashboard/backend/config.py index 1d7475e..02b6f1c 100644 --- a/dashboard/backend/config.py +++ b/dashboard/backend/config.py @@ -44,6 +44,9 @@ CORS_ORIGINS = os.environ.get("CORS_ORIGINS", "https://nas.jimmygan.com,https:// # Chat Summary CHAT_SUMMARY_DB = os.environ.get("CHAT_SUMMARY_DB", "/app/data/chat-summarizer/chat_summary.db") +# Info Engine +INFO_ENGINE_DB = os.environ.get("INFO_ENGINE_DB", "/app/data/info-engine/info_engine.db") + # LiteLLM LITELLM_URL = os.environ.get("LITELLM_URL", "http://127.0.0.1:4005") LITELLM_HEALTH_API_KEY = os.environ.get("LITELLM_HEALTH_API_KEY", "") diff --git a/dashboard/backend/main.py b/dashboard/backend/main.py index eb59587..7048a63 100644 --- a/dashboard/backend/main.py +++ b/dashboard/backend/main.py @@ -6,7 +6,7 @@ from fastapi.responses import JSONResponse from slowapi import Limiter from slowapi.util import get_remote_address from slowapi.errors import RateLimitExceeded -from routers import docker_router, gitea, files, terminal, system, auth, chat_summary, security, passkey, totp, litellm, cc_connect +from routers import docker_router, gitea, files, terminal, system, auth, chat_summary, security, passkey, totp, litellm, cc_connect, info_engine import auth as auth_module from rbac import require_page, require_write @@ -185,6 +185,8 @@ app.include_router(cc_connect.router, prefix="/api/cc-connect", dependencies=[Depends(_inject_user), Depends(require_page("dashboard"))]) app.include_router(chat_summary.router, prefix="/api/chat-summary", dependencies=[Depends(_inject_user), Depends(require_page("chat-digest"))]) +app.include_router(info_engine.router, prefix="/api/info-engine", + dependencies=[Depends(_inject_user), Depends(require_page("dashboard"))]) app.include_router(security.router, prefix="/api/security", dependencies=[Depends(_inject_user), Depends(require_page("security"))]) diff --git a/dashboard/backend/routers/info_engine.py b/dashboard/backend/routers/info_engine.py new file mode 100644 index 0000000..b9e8d6a --- /dev/null +++ b/dashboard/backend/routers/info_engine.py @@ -0,0 +1,111 @@ +import json + +import aiosqlite +from fastapi import APIRouter, HTTPException, Query + +from config import INFO_ENGINE_DB + +router = APIRouter() + + +async def _db(): + return aiosqlite.connect(INFO_ENGINE_DB) + + +def _row_to_item(row): + tags = [] + if row[5]: + try: + tags = json.loads(row[5]) + except Exception: + tags = [] + + return { + "id": row[0], + "source": row[1], + "title": row[2], + "summary": row[3], + "url": row[4], + "tags": tags, + "published_at": row[6], + "collected_at": row[7], + "content_text": row[8], + "content_html": row[9], + } + + +@router.get("/items") +async def list_items( + limit: int = Query(50, ge=1, le=200), + offset: int = Query(0, ge=0), + source: str = Query(""), + tag: str = Query(""), + since: str = Query(""), +): + where = [] + params = [] + + if source: + where.append("source = ?") + params.append(source) + if tag: + where.append("tags LIKE ?") + params.append(f'%"{tag}"%') + if since: + where.append("collected_at >= ?") + params.append(since) + + where_clause = f"WHERE {' AND '.join(where)}" if where else "" + + async with await _db() as db: + count_cursor = await db.execute( + f"SELECT COUNT(*) FROM info_items {where_clause}", + tuple(params), + ) + total = (await count_cursor.fetchone())[0] + + cursor = await db.execute( + f""" + SELECT id, source, title, summary, url, tags, published_at, collected_at, content_text, content_html + FROM info_items + {where_clause} + ORDER BY COALESCE(published_at, collected_at) DESC, id DESC + LIMIT ? OFFSET ? + """, + tuple(params + [limit, offset]), + ) + rows = await cursor.fetchall() + + return {"items": [_row_to_item(row) for row in rows], "total": total} + + +@router.get("/items/{item_id}") +async def get_item(item_id: int): + async with await _db() as db: + cursor = await db.execute( + """ + SELECT id, source, title, summary, url, tags, published_at, collected_at, content_text, content_html + FROM info_items + WHERE id = ? + """, + (item_id,), + ) + row = await cursor.fetchone() + + if not row: + raise HTTPException(404, "Item not found") + + return _row_to_item(row) + + +@router.get("/health") +async def health(): + async with await _db() as db: + cursor = await db.execute("SELECT COUNT(*), MAX(collected_at) FROM info_items") + row = await cursor.fetchone() + + return { + "status": "ok", + "item_count": row[0] if row else 0, + "last_collect_at": row[1] if row else None, + } diff --git a/dashboard/docker-compose.yml b/dashboard/docker-compose.yml index 3808068..fab1838 100644 --- a/dashboard/docker-compose.yml +++ b/dashboard/docker-compose.yml @@ -50,6 +50,7 @@ services: - /volume1/docker/nas-dashboard/ssh/vps_terminal:/app/ssh/vps_id_ed25519:ro - /volume1/docker/nas-dashboard/ssh/known_hosts:/app/ssh/known_hosts:ro - /volume1/docker/chat-summarizer/data:/app/data/chat-summarizer:ro + - /volume1/docker/info-engine/data:/app/data/info-engine:ro - /volume1/docker/nas-dashboard/rbac.json:/volume1/docker/nas-dashboard/rbac.json healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/api/health')"] diff --git a/dashboard/frontend/src/App.svelte b/dashboard/frontend/src/App.svelte index cdc57b6..3d3a533 100644 --- a/dashboard/frontend/src/App.svelte +++ b/dashboard/frontend/src/App.svelte @@ -11,6 +11,7 @@ import Security from "./routes/Security.svelte"; import LiteLLM from "./routes/LiteLLM.svelte"; import CcConnect from "./routes/CcConnect.svelte"; + import InfoEngine from "./routes/InfoEngine.svelte"; import Login from "./routes/Login.svelte"; import { onMount } from "svelte"; import { getToken, checkAuth, setToken, setRefreshToken, currentUser, setCurrentUser, getPreferences, savePreferences } from "./lib/api.js"; @@ -159,6 +160,8 @@ {:else if page === "cc-connect" && hasPageAccess("dashboard")} + {:else if page === "info-engine" && hasPageAccess("dashboard")} + {:else} {/if} diff --git a/dashboard/frontend/src/components/Sidebar.svelte b/dashboard/frontend/src/components/Sidebar.svelte index 99d025b..e9fd1a5 100644 --- a/dashboard/frontend/src/components/Sidebar.svelte +++ b/dashboard/frontend/src/components/Sidebar.svelte @@ -4,7 +4,9 @@ function canAccess(id) { if (!id) return true; if (allowedPages === "*") return true; - return Array.isArray(allowedPages) && allowedPages.includes(id); + if (!Array.isArray(allowedPages)) return false; + if (id === "info-engine") return allowedPages.includes("dashboard"); + return allowedPages.includes(id); } function canSeeSidebarLink(item) { @@ -16,6 +18,7 @@ const defaultLinks = [ { id: "dashboard", label: "Overview", icon: "grid" }, + { id: "info-engine", label: "Info Engine", icon: "sparkles" }, { id: "litellm", label: "LiteLLM", icon: "bolt" }, { id: "docker", label: "Docker", icon: "box" }, { id: "files", label: "Files", icon: "folder" }, @@ -228,6 +231,8 @@ {#if icon === "grid"} + {:else if icon === "sparkles"} + {:else if icon === "box"} {:else if icon === "bolt"} diff --git a/dashboard/frontend/src/lib/api.js b/dashboard/frontend/src/lib/api.js index 8a169b7..2c72c26 100644 --- a/dashboard/frontend/src/lib/api.js +++ b/dashboard/frontend/src/lib/api.js @@ -147,6 +147,21 @@ export function getCcConnectHealth() { return get("/cc-connect/health"); } +export function getInfoEngineItems(params = {}) { + const query = new URLSearchParams(); + if (params.limit !== undefined) query.set("limit", String(params.limit)); + if (params.offset !== undefined) query.set("offset", String(params.offset)); + if (params.source) query.set("source", params.source); + if (params.tag) query.set("tag", params.tag); + if (params.since) query.set("since", params.since); + const suffix = query.toString() ? `?${query.toString()}` : ""; + return get(`/info-engine/items${suffix}`); +} + +export function getInfoEngineItem(id) { + return get(`/info-engine/items/${id}`); +} + export function put(path, data) { return request(path, { method: "PUT", json: data }); } diff --git a/dashboard/frontend/src/routes/InfoEngine.svelte b/dashboard/frontend/src/routes/InfoEngine.svelte new file mode 100644 index 0000000..25d7433 --- /dev/null +++ b/dashboard/frontend/src/routes/InfoEngine.svelte @@ -0,0 +1,139 @@ + + +
+
+
+

Info Engine

+

Context-aware stream of curated signals

+
+ +
+ + {#if requestError} +
+

{requestError}

+
+ {/if} + +
+
+
+ Latest items ({total}) +
+ {#if loading} +
+ {#each Array(6) as _} +
+ {/each} +
+ {:else if items.length === 0} +
No items yet.
+ {:else} +
+ {#each items as item} + + {/each} +
+ {/if} +
+ +
+ {#if loadingDetail} +
+
+
+
+
+
+ {:else if selectedItem} +

{selectedItem.title}

+
+

Source: {selectedItem.source}

+

Published: {formatDate(selectedItem.published_at)}

+

Collected: {formatDate(selectedItem.collected_at)}

+
+ + {#if selectedItem.tags?.length} +
+ {#each selectedItem.tags as tag} + {tag} + {/each} +
+ {/if} + +

{selectedItem.summary || selectedItem.content_text || "No summary available."}

+ + + Open source + + {:else} +

Select an item to view details.

+ {/if} +
+
+
diff --git a/info-engine/Dockerfile b/info-engine/Dockerfile new file mode 100644 index 0000000..3613a77 --- /dev/null +++ b/info-engine/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.12-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +CMD ["python", "-u", "main.py"] diff --git a/info-engine/collector.py b/info-engine/collector.py new file mode 100644 index 0000000..373c0ff --- /dev/null +++ b/info-engine/collector.py @@ -0,0 +1,151 @@ +import logging +import os +from datetime import datetime +from email.utils import parsedate_to_datetime +from urllib.parse import urlparse +import xml.etree.ElementTree as ET + +import httpx + +log = logging.getLogger(__name__) + + +def _source_name(url: str) -> str: + host = urlparse(url).netloc + return host or url + + +def _first_text(parent, paths: list[str]) -> str: + for path in paths: + node = parent.find(path) + if node is not None and node.text: + value = node.text.strip() + if value: + return value + return "" + + +def _atom_link(entry) -> str: + for link in entry.findall("{http://www.w3.org/2005/Atom}link"): + href = (link.attrib.get("href") or "").strip() + rel = (link.attrib.get("rel") or "alternate").strip() + if href and rel == "alternate": + return href + links = entry.findall("{http://www.w3.org/2005/Atom}link") + if links: + return (links[0].attrib.get("href") or "").strip() + return "" + + +def _parse_datetime(value: str | None) -> str | None: + if not value: + return None + value = value.strip() + if not value: + return None + + try: + return parsedate_to_datetime(value).isoformat() + except Exception: + pass + + try: + iso = value.replace("Z", "+00:00") + return datetime.fromisoformat(iso).isoformat() + except Exception: + return None + + +def _parse_feed(feed_url: str, xml_text: str) -> list[dict]: + root = ET.fromstring(xml_text) + source = _source_name(feed_url) + + items = [] + + channel = root.find("channel") + if channel is not None: + for item in channel.findall("item"): + title = _first_text(item, ["title"]) + link = _first_text(item, ["link"]) + summary = _first_text(item, ["description"]) + pub = _first_text(item, ["pubDate", "date"]) + tags = [n.text.strip() for n in item.findall("category") if n.text and n.text.strip()] + if title and link: + items.append( + { + "source": source, + "title": title, + "summary": summary, + "url": link, + "tags": tags, + "published_at": _parse_datetime(pub), + "content_text": summary, + "content_html": summary, + } + ) + return items + + for entry in root.findall("{http://www.w3.org/2005/Atom}entry"): + title = _first_text(entry, ["{http://www.w3.org/2005/Atom}title"]) + link = _atom_link(entry) + summary = _first_text( + entry, + [ + "{http://www.w3.org/2005/Atom}summary", + "{http://www.w3.org/2005/Atom}content", + ], + ) + pub = _first_text( + entry, + [ + "{http://www.w3.org/2005/Atom}published", + "{http://www.w3.org/2005/Atom}updated", + ], + ) + tags = [ + (cat.attrib.get("term") or "").strip() + for cat in entry.findall("{http://www.w3.org/2005/Atom}category") + if (cat.attrib.get("term") or "").strip() + ] + if title and link: + items.append( + { + "source": source, + "title": title, + "summary": summary, + "url": link, + "tags": tags, + "published_at": _parse_datetime(pub), + "content_text": summary, + "content_html": summary, + } + ) + + return items + + +async def collect_sources() -> list[dict]: + sources_env = os.environ.get( + "INFO_ENGINE_SOURCES", + "https://hnrss.org/frontpage,https://lobste.rs/rss", + ) + sources = [s.strip() for s in sources_env.split(",") if s.strip()] + if not sources: + log.warning("No INFO_ENGINE_SOURCES configured") + return [] + + timeout = httpx.Timeout(15.0) + all_items = [] + + async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client: + for source_url in sources: + try: + response = await client.get(source_url) + response.raise_for_status() + parsed = _parse_feed(source_url, response.text) + all_items.extend(parsed) + log.info("Collected %d items from %s", len(parsed), source_url) + except Exception as exc: + log.error("Failed to collect from %s: %s", source_url, exc) + + return all_items diff --git a/info-engine/db.py b/info-engine/db.py new file mode 100644 index 0000000..72f2b94 --- /dev/null +++ b/info-engine/db.py @@ -0,0 +1,64 @@ +import json +import os +from datetime import datetime, timezone + +import aiosqlite + +DB_PATH = os.environ.get("DB_PATH", "/app/data/info_engine.db") + + +async def init_db(): + async with aiosqlite.connect(DB_PATH) as db: + await db.execute( + """ + CREATE TABLE IF NOT EXISTS info_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL, + title TEXT NOT NULL, + summary TEXT NOT NULL, + url TEXT NOT NULL UNIQUE, + tags TEXT NOT NULL, + published_at DATETIME, + collected_at DATETIME NOT NULL, + content_text TEXT, + content_html TEXT + ) + """ + ) + await db.execute("CREATE INDEX IF NOT EXISTS idx_info_items_collected_at ON info_items(collected_at DESC)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_info_items_published_at ON info_items(published_at DESC)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_info_items_source ON info_items(source)") + await db.commit() + + +async def upsert_items(items: list[dict]) -> int: + if not items: + return 0 + + inserted = 0 + now = datetime.now(timezone.utc).isoformat() + + async with aiosqlite.connect(DB_PATH) as db: + for item in items: + cursor = await db.execute( + """ + INSERT OR IGNORE INTO info_items + (source, title, summary, url, tags, published_at, collected_at, content_text, content_html) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + item["source"], + item["title"], + item.get("summary", ""), + item["url"], + json.dumps(item.get("tags", []), ensure_ascii=False), + item.get("published_at"), + now, + item.get("content_text"), + item.get("content_html"), + ), + ) + inserted += cursor.rowcount + await db.commit() + + return inserted diff --git a/info-engine/docker-compose.yml b/info-engine/docker-compose.yml new file mode 100644 index 0000000..488fead --- /dev/null +++ b/info-engine/docker-compose.yml @@ -0,0 +1,16 @@ +services: + info-engine: + build: . + container_name: info-engine + restart: unless-stopped + volumes: + - /volume1/docker/info-engine/data:/app/data + environment: + - DB_PATH=/app/data/info_engine.db + - INFO_ENGINE_INTERVAL_SECONDS=${INFO_ENGINE_INTERVAL_SECONDS:-900} + - INFO_ENGINE_SOURCES=${INFO_ENGINE_SOURCES:-https://hnrss.org/frontpage,https://lobste.rs/rss} + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" diff --git a/info-engine/main.py b/info-engine/main.py new file mode 100644 index 0000000..9728d3a --- /dev/null +++ b/info-engine/main.py @@ -0,0 +1,37 @@ +import asyncio +import logging +import os + +from collector import collect_sources +from db import init_db, upsert_items +from processor import process_items + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) + +COLLECT_INTERVAL_SECONDS = int(os.environ.get("INFO_ENGINE_INTERVAL_SECONDS", "900")) + + +async def run_once() -> int: + raw_items = await collect_sources() + processed = process_items(raw_items) + inserted = await upsert_items(processed) + return inserted + + +async def main(): + await init_db() + log.info("Info engine started (interval=%ss)", COLLECT_INTERVAL_SECONDS) + + while True: + try: + inserted = await run_once() + log.info("Info engine cycle complete (inserted=%d)", inserted) + except Exception as exc: + log.error("Info engine cycle failed: %s", exc) + + await asyncio.sleep(COLLECT_INTERVAL_SECONDS) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/info-engine/processor.py b/info-engine/processor.py new file mode 100644 index 0000000..181e1b0 --- /dev/null +++ b/info-engine/processor.py @@ -0,0 +1,45 @@ +from urllib.parse import urldefrag, urlparse + + +def _normalize_url(url: str) -> str: + clean, _ = urldefrag(url.strip()) + parsed = urlparse(clean) + if parsed.scheme and parsed.netloc: + return clean + return "" + + +def process_items(items: list[dict]) -> list[dict]: + seen = set() + processed = [] + + for item in items: + title = (item.get("title") or "").strip() + url = _normalize_url(item.get("url") or "") + if not title or not url: + continue + + dedupe_key = url.lower() + if dedupe_key in seen: + continue + seen.add(dedupe_key) + + summary = (item.get("summary") or "").strip() + tags = item.get("tags") or [] + if not isinstance(tags, list): + tags = [] + + processed.append( + { + "source": (item.get("source") or "unknown").strip() or "unknown", + "title": title, + "summary": summary, + "url": url, + "tags": [str(t).strip() for t in tags if str(t).strip()], + "published_at": item.get("published_at"), + "content_text": item.get("content_text") or summary, + "content_html": item.get("content_html") or summary, + } + ) + + return processed diff --git a/info-engine/requirements.txt b/info-engine/requirements.txt new file mode 100644 index 0000000..42d058a --- /dev/null +++ b/info-engine/requirements.txt @@ -0,0 +1,2 @@ +aiosqlite +httpx