dashboard/backend/config.py

@@ -44,6 +44,9 @@ CORS_ORIGINS = os.environ.get("CORS_ORIGINS", "https://nas.jimmygan.com,https://
 # Chat Summary
 CHAT_SUMMARY_DB = os.environ.get("CHAT_SUMMARY_DB", "/app/data/chat-summarizer/chat_summary.db")
 
+# Info Engine
+INFO_ENGINE_DB = os.environ.get("INFO_ENGINE_DB", "/app/data/info-engine/info_engine.db")
+
 # LiteLLM
 LITELLM_URL = os.environ.get("LITELLM_URL", "http://127.0.0.1:4005")
 LITELLM_HEALTH_API_KEY = os.environ.get("LITELLM_HEALTH_API_KEY", "")

dashboard/backend/main.py

@@ -6,7 +6,7 @@ from fastapi.responses import JSONResponse
 from slowapi import Limiter
 from slowapi.util import get_remote_address
 from slowapi.errors import RateLimitExceeded
-from routers import docker_router, gitea, files, terminal, system, auth, chat_summary, security, passkey, totp, litellm, cc_connect
+from routers import docker_router, gitea, files, terminal, system, auth, chat_summary, security, passkey, totp, litellm, cc_connect, info_engine
 import auth as auth_module
 from rbac import require_page, require_write
 
@@ -185,6 +185,8 @@ app.include_router(cc_connect.router, prefix="/api/cc-connect",
     dependencies=[Depends(_inject_user), Depends(require_page("dashboard"))])
 app.include_router(chat_summary.router, prefix="/api/chat-summary",
     dependencies=[Depends(_inject_user), Depends(require_page("chat-digest"))])
+app.include_router(info_engine.router, prefix="/api/info-engine",
+    dependencies=[Depends(_inject_user), Depends(require_page("dashboard"))])
 app.include_router(security.router, prefix="/api/security",
     dependencies=[Depends(_inject_user), Depends(require_page("security"))])
 

dashboard/backend/routers/info_engine.py

@@ -0,0 +1,111 @@
+import json
+
+import aiosqlite
+from fastapi import APIRouter, HTTPException, Query
+
+from config import INFO_ENGINE_DB
+
+router = APIRouter()
+
+
+async def _db():
+    return aiosqlite.connect(INFO_ENGINE_DB)
+
+
+def _row_to_item(row):
+    tags = []
+    if row[5]:
+        try:
+            tags = json.loads(row[5])
+        except Exception:
+            tags = []
+
+    return {
+        "id": row[0],
+        "source": row[1],
+        "title": row[2],
+        "summary": row[3],
+        "url": row[4],
+        "tags": tags,
+        "published_at": row[6],
+        "collected_at": row[7],
+        "content_text": row[8],
+        "content_html": row[9],
+    }
+
+
+@router.get("/items")
+async def list_items(
+    limit: int = Query(50, ge=1, le=200),
+    offset: int = Query(0, ge=0),
+    source: str = Query(""),
+    tag: str = Query(""),
+    since: str = Query(""),
+):
+    where = []
+    params = []
+
+    if source:
+        where.append("source = ?")
+        params.append(source)
+    if tag:
+        where.append("tags LIKE ?")
+        params.append(f'%"{tag}"%')
+    if since:
+        where.append("collected_at >= ?")
+        params.append(since)
+
+    where_clause = f"WHERE {' AND '.join(where)}" if where else ""
+
+    async with await _db() as db:
+        count_cursor = await db.execute(
+            f"SELECT COUNT(*) FROM info_items {where_clause}",
+            tuple(params),
+        )
+        total = (await count_cursor.fetchone())[0]
+
+        cursor = await db.execute(
+            f"""
+            SELECT id, source, title, summary, url, tags, published_at, collected_at, content_text, content_html
+            FROM info_items
+            {where_clause}
+            ORDER BY COALESCE(published_at, collected_at) DESC, id DESC
+            LIMIT ? OFFSET ?
+            """,
+            tuple(params + [limit, offset]),
+        )
+        rows = await cursor.fetchall()
+
+    return {"items": [_row_to_item(row) for row in rows], "total": total}
+
+
+@router.get("/items/{item_id}")
+async def get_item(item_id: int):
+    async with await _db() as db:
+        cursor = await db.execute(
+            """
+            SELECT id, source, title, summary, url, tags, published_at, collected_at, content_text, content_html
+            FROM info_items
+            WHERE id = ?
+            """,
+            (item_id,),
+        )
+        row = await cursor.fetchone()
+
+    if not row:
+        raise HTTPException(404, "Item not found")
+
+    return _row_to_item(row)
+
+
+@router.get("/health")
+async def health():
+    async with await _db() as db:
+        cursor = await db.execute("SELECT COUNT(*), MAX(collected_at) FROM info_items")
+        row = await cursor.fetchone()
+
+    return {
+        "status": "ok",
+        "item_count": row[0] if row else 0,
+        "last_collect_at": row[1] if row else None,
+    }

dashboard/docker-compose.yml

@@ -50,6 +50,7 @@ services:
       - /volume1/docker/nas-dashboard/ssh/vps_terminal:/app/ssh/vps_id_ed25519:ro
       - /volume1/docker/nas-dashboard/ssh/known_hosts:/app/ssh/known_hosts:ro
       - /volume1/docker/chat-summarizer/data:/app/data/chat-summarizer:ro
+      - /volume1/docker/info-engine/data:/app/data/info-engine:ro
       - /volume1/docker/nas-dashboard/rbac.json:/volume1/docker/nas-dashboard/rbac.json
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/api/health')"]

dashboard/frontend/src/App.svelte

@@ -11,6 +11,7 @@
   import Security from "./routes/Security.svelte";
   import LiteLLM from "./routes/LiteLLM.svelte";
   import CcConnect from "./routes/CcConnect.svelte";
+  import InfoEngine from "./routes/InfoEngine.svelte";
   import Login from "./routes/Login.svelte";
   import { onMount } from "svelte";
   import { getToken, checkAuth, setToken, setRefreshToken, currentUser, setCurrentUser, getPreferences, savePreferences } from "./lib/api.js";
@@ -159,6 +160,8 @@
           <LiteLLM />
         {:else if page === "cc-connect" && hasPageAccess("dashboard")}
           <CcConnect />
+        {:else if page === "info-engine" && hasPageAccess("dashboard")}
+          <InfoEngine />
         {:else}
           <Dashboard />
         {/if}

dashboard/frontend/src/components/Sidebar.svelte

@@ -4,7 +4,9 @@
   function canAccess(id) {
     if (!id) return true;
     if (allowedPages === "*") return true;
-    return Array.isArray(allowedPages) && allowedPages.includes(id);
+    if (!Array.isArray(allowedPages)) return false;
+    if (id === "info-engine") return allowedPages.includes("dashboard");
+    return allowedPages.includes(id);
   }
 
   function canSeeSidebarLink(item) {
@@ -16,6 +18,7 @@
 
   const defaultLinks = [
     { id: "dashboard", label: "Overview", icon: "grid" },
+    { id: "info-engine", label: "Info Engine", icon: "sparkles" },
     { id: "litellm", label: "LiteLLM", icon: "bolt" },
     { id: "docker", label: "Docker", icon: "box" },
     { id: "files", label: "Files", icon: "folder" },
@@ -228,6 +231,8 @@
   <span class="w-5 h-5 flex items-center justify-center">
     {#if icon === "grid"}
       <svg class="w-[18px] h-[18px]" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="1.8"><path stroke-linecap="round" stroke-linejoin="round" d="M4 6a2 2 0 012-2h2a2 2 0 012 2v2a2 2 0 01-2 2H6a2 2 0 01-2-2V6zm10 0a2 2 0 012-2h2a2 2 0 012 2v2a2 2 0 01-2 2h-2a2 2 0 01-2-2V6zM4 16a2 2 0 012-2h2a2 2 0 012 2v2a2 2 0 01-2 2H6a2 2 0 01-2-2v-2zm10 0a2 2 0 012-2h2a2 2 0 012 2v2a2 2 0 01-2 2h-2a2 2 0 01-2-2v-2z" /></svg>
+    {:else if icon === "sparkles"}
+      <svg class="w-[18px] h-[18px]" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="1.8"><path stroke-linecap="round" stroke-linejoin="round" d="M12 3l1.8 4.2L18 9l-4.2 1.8L12 15l-1.8-4.2L6 9l4.2-1.8L12 3zm6 11l.9 2.1L21 17l-2.1.9L18 20l-.9-2.1L15 17l2.1-.9L18 14zM5 14l.9 2.1L8 17l-2.1.9L5 20l-.9-2.1L2 17l2.1-.9L5 14z" /></svg>
     {:else if icon === "box"}
       <svg class="w-[18px] h-[18px]" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="1.8"><path stroke-linecap="round" stroke-linejoin="round" d="M20 7l-8-4-8 4m16 0l-8 4m8-4v10l-8 4m0-10L4 7m8 4v10M4 7v10l8 4" /></svg>
     {:else if icon === "bolt"}

dashboard/frontend/src/lib/api.js

@@ -147,6 +147,21 @@ export function getCcConnectHealth() {
   return get("/cc-connect/health");
 }
 
+export function getInfoEngineItems(params = {}) {
+  const query = new URLSearchParams();
+  if (params.limit !== undefined) query.set("limit", String(params.limit));
+  if (params.offset !== undefined) query.set("offset", String(params.offset));
+  if (params.source) query.set("source", params.source);
+  if (params.tag) query.set("tag", params.tag);
+  if (params.since) query.set("since", params.since);
+  const suffix = query.toString() ? `?${query.toString()}` : "";
+  return get(`/info-engine/items${suffix}`);
+}
+
+export function getInfoEngineItem(id) {
+  return get(`/info-engine/items/${id}`);
+}
+
 export function put(path, data) {
   return request(path, { method: "PUT", json: data });
 }

dashboard/frontend/src/routes/InfoEngine.svelte

@@ -0,0 +1,139 @@
+<script>
+  import { onMount } from "svelte";
+  import { getInfoEngineItems, getInfoEngineItem } from "../lib/api.js";
+
+  let loading = $state(true);
+  let loadingDetail = $state(false);
+  let requestError = $state("");
+  let items = $state([]);
+  let total = $state(0);
+  let selectedId = $state(null);
+  let selectedItem = $state(null);
+
+  async function loadItems() {
+    loading = true;
+    requestError = "";
+    try {
+      const res = await getInfoEngineItems({ limit: 50, offset: 0 });
+      items = res.items || [];
+      total = res.total || 0;
+      if (items.length > 0) {
+        await selectItem(items[0].id);
+      } else {
+        selectedId = null;
+        selectedItem = null;
+      }
+    } catch (e) {
+      requestError = e?.body?.detail || e?.message || "Failed to load items";
+      items = [];
+      total = 0;
+      selectedId = null;
+      selectedItem = null;
+    } finally {
+      loading = false;
+    }
+  }
+
+  async function selectItem(id) {
+    selectedId = id;
+    loadingDetail = true;
+    requestError = "";
+    try {
+      selectedItem = await getInfoEngineItem(id);
+    } catch (e) {
+      selectedItem = null;
+      requestError = e?.body?.detail || e?.message || "Failed to load item";
+    } finally {
+      loadingDetail = false;
+    }
+  }
+
+  function formatDate(value) {
+    if (!value) return "—";
+    try {
+      return new Date(value).toLocaleString();
+    } catch {
+      return value;
+    }
+  }
+
+  onMount(loadItems);
+</script>
+
+<div class="space-y-6">
+  <div class="flex items-center justify-between">
+    <div>
+      <h1 class="text-2xl font-bold text-surface-900 dark:text-white tracking-tight">Info Engine</h1>
+      <p class="text-sm text-surface-400 mt-1">Context-aware stream of curated signals</p>
+    </div>
+    <button onclick={loadItems} disabled={loading} class="px-3 py-1.5 text-xs font-medium text-primary-600 bg-primary-50 hover:bg-primary-100 rounded-lg transition-colors disabled:opacity-50 dark:bg-primary-900/30 dark:text-primary-300">
+      {loading ? "Loading..." : "Refresh"}
+    </button>
+  </div>
+
+  {#if requestError}
+    <div class="bg-rose-50 dark:bg-rose-900/20 border border-rose-200 dark:border-rose-700 rounded-xl p-4 shadow-sm">
+      <p class="text-sm font-medium text-rose-700 dark:text-rose-300">{requestError}</p>
+    </div>
+  {/if}
+
+  <div class="grid grid-cols-1 lg:grid-cols-3 gap-4">
+    <div class="lg:col-span-1 bg-white dark:bg-surface-800 rounded-xl border border-surface-200 dark:border-surface-700 shadow-sm">
+      <div class="px-4 py-3 border-b border-surface-200 dark:border-surface-700 text-xs text-surface-400">
+        Latest items ({total})
+      </div>
+      {#if loading}
+        <div class="p-4 space-y-2">
+          {#each Array(6) as _}
+            <div class="h-12 rounded-lg bg-surface-100 dark:bg-surface-700 animate-pulse"></div>
+          {/each}
+        </div>
+      {:else if items.length === 0}
+        <div class="p-4 text-sm text-surface-400">No items yet.</div>
+      {:else}
+        <div class="max-h-[560px] overflow-y-auto p-2">
+          {#each items as item}
+            <button onclick={() => selectItem(item.id)} class="w-full text-left px-3 py-2 rounded-lg mb-1 transition-colors {selectedId === item.id ? 'bg-primary-50 dark:bg-primary-900/30' : 'hover:bg-surface-100 dark:hover:bg-surface-700'}">
+              <p class="text-sm font-semibold text-surface-800 dark:text-surface-100 line-clamp-2">{item.title}</p>
+              <p class="text-xs text-surface-400 mt-1">{item.source}</p>
+            </button>
+          {/each}
+        </div>
+      {/if}
+    </div>
+
+    <div class="lg:col-span-2 bg-white dark:bg-surface-800 rounded-xl border border-surface-200 dark:border-surface-700 p-5 shadow-sm">
+      {#if loadingDetail}
+        <div class="space-y-3">
+          <div class="h-7 w-3/4 rounded bg-surface-100 dark:bg-surface-700 animate-pulse"></div>
+          <div class="h-4 w-full rounded bg-surface-100 dark:bg-surface-700 animate-pulse"></div>
+          <div class="h-4 w-5/6 rounded bg-surface-100 dark:bg-surface-700 animate-pulse"></div>
+          <div class="h-24 w-full rounded bg-surface-100 dark:bg-surface-700 animate-pulse"></div>
+        </div>
+      {:else if selectedItem}
+        <h2 class="text-xl font-bold text-surface-900 dark:text-white">{selectedItem.title}</h2>
+        <div class="mt-2 text-xs text-surface-400 space-y-1">
+          <p>Source: {selectedItem.source}</p>
+          <p>Published: {formatDate(selectedItem.published_at)}</p>
+          <p>Collected: {formatDate(selectedItem.collected_at)}</p>
+        </div>
+
+        {#if selectedItem.tags?.length}
+          <div class="mt-3 flex flex-wrap gap-2">
+            {#each selectedItem.tags as tag}
+              <span class="px-2 py-0.5 text-xs rounded-full bg-surface-100 text-surface-600 dark:bg-surface-700 dark:text-surface-300">{tag}</span>
+            {/each}
+          </div>
+        {/if}
+
+        <p class="mt-4 text-sm text-surface-700 dark:text-surface-300 whitespace-pre-wrap">{selectedItem.summary || selectedItem.content_text || "No summary available."}</p>
+
+        <a href={selectedItem.url} target="_blank" rel="noopener" class="inline-flex mt-4 px-3 py-1.5 text-xs font-medium rounded-lg bg-primary-600 text-white hover:bg-primary-700">
+          Open source
+        </a>
+      {:else}
+        <p class="text-sm text-surface-400">Select an item to view details.</p>
+      {/if}
+    </div>
+  </div>
+</div>

info-engine/Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+COPY . .
+CMD ["python", "-u", "main.py"]

info-engine/collector.py

@@ -0,0 +1,151 @@
+import logging
+import os
+from datetime import datetime
+from email.utils import parsedate_to_datetime
+from urllib.parse import urlparse
+import xml.etree.ElementTree as ET
+
+import httpx
+
+log = logging.getLogger(__name__)
+
+
+def _source_name(url: str) -> str:
+    host = urlparse(url).netloc
+    return host or url
+
+
+def _first_text(parent, paths: list[str]) -> str:
+    for path in paths:
+        node = parent.find(path)
+        if node is not None and node.text:
+            value = node.text.strip()
+            if value:
+                return value
+    return ""
+
+
+def _atom_link(entry) -> str:
+    for link in entry.findall("{http://www.w3.org/2005/Atom}link"):
+        href = (link.attrib.get("href") or "").strip()
+        rel = (link.attrib.get("rel") or "alternate").strip()
+        if href and rel == "alternate":
+            return href
+    links = entry.findall("{http://www.w3.org/2005/Atom}link")
+    if links:
+        return (links[0].attrib.get("href") or "").strip()
+    return ""
+
+
+def _parse_datetime(value: str | None) -> str | None:
+    if not value:
+        return None
+    value = value.strip()
+    if not value:
+        return None
+
+    try:
+        return parsedate_to_datetime(value).isoformat()
+    except Exception:
+        pass
+
+    try:
+        iso = value.replace("Z", "+00:00")
+        return datetime.fromisoformat(iso).isoformat()
+    except Exception:
+        return None
+
+
+def _parse_feed(feed_url: str, xml_text: str) -> list[dict]:
+    root = ET.fromstring(xml_text)
+    source = _source_name(feed_url)
+
+    items = []
+
+    channel = root.find("channel")
+    if channel is not None:
+        for item in channel.findall("item"):
+            title = _first_text(item, ["title"])
+            link = _first_text(item, ["link"])
+            summary = _first_text(item, ["description"])
+            pub = _first_text(item, ["pubDate", "date"])
+            tags = [n.text.strip() for n in item.findall("category") if n.text and n.text.strip()]
+            if title and link:
+                items.append(
+                    {
+                        "source": source,
+                        "title": title,
+                        "summary": summary,
+                        "url": link,
+                        "tags": tags,
+                        "published_at": _parse_datetime(pub),
+                        "content_text": summary,
+                        "content_html": summary,
+                    }
+                )
+        return items
+
+    for entry in root.findall("{http://www.w3.org/2005/Atom}entry"):
+        title = _first_text(entry, ["{http://www.w3.org/2005/Atom}title"])
+        link = _atom_link(entry)
+        summary = _first_text(
+            entry,
+            [
+                "{http://www.w3.org/2005/Atom}summary",
+                "{http://www.w3.org/2005/Atom}content",
+            ],
+        )
+        pub = _first_text(
+            entry,
+            [
+                "{http://www.w3.org/2005/Atom}published",
+                "{http://www.w3.org/2005/Atom}updated",
+            ],
+        )
+        tags = [
+            (cat.attrib.get("term") or "").strip()
+            for cat in entry.findall("{http://www.w3.org/2005/Atom}category")
+            if (cat.attrib.get("term") or "").strip()
+        ]
+        if title and link:
+            items.append(
+                {
+                    "source": source,
+                    "title": title,
+                    "summary": summary,
+                    "url": link,
+                    "tags": tags,
+                    "published_at": _parse_datetime(pub),
+                    "content_text": summary,
+                    "content_html": summary,
+                }
+            )
+
+    return items
+
+
+async def collect_sources() -> list[dict]:
+    sources_env = os.environ.get(
+        "INFO_ENGINE_SOURCES",
+        "https://hnrss.org/frontpage,https://lobste.rs/rss",
+    )
+    sources = [s.strip() for s in sources_env.split(",") if s.strip()]
+    if not sources:
+        log.warning("No INFO_ENGINE_SOURCES configured")
+        return []
+
+    timeout = httpx.Timeout(15.0)
+    all_items = []
+
+    async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
+        for source_url in sources:
+            try:
+                response = await client.get(source_url)
+                response.raise_for_status()
+                parsed = _parse_feed(source_url, response.text)
+                all_items.extend(parsed)
+                log.info("Collected %d items from %s", len(parsed), source_url)
+            except Exception as exc:
+                log.error("Failed to collect from %s: %s", source_url, exc)
+
+    return all_items

info-engine/db.py

@@ -0,0 +1,64 @@
+import json
+import os
+from datetime import datetime, timezone
+
+import aiosqlite
+
+DB_PATH = os.environ.get("DB_PATH", "/app/data/info_engine.db")
+
+
+async def init_db():
+    async with aiosqlite.connect(DB_PATH) as db:
+        await db.execute(
+            """
+            CREATE TABLE IF NOT EXISTS info_items (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                source TEXT NOT NULL,
+                title TEXT NOT NULL,
+                summary TEXT NOT NULL,
+                url TEXT NOT NULL UNIQUE,
+                tags TEXT NOT NULL,
+                published_at DATETIME,
+                collected_at DATETIME NOT NULL,
+                content_text TEXT,
+                content_html TEXT
+            )
+            """
+        )
+        await db.execute("CREATE INDEX IF NOT EXISTS idx_info_items_collected_at ON info_items(collected_at DESC)")
+        await db.execute("CREATE INDEX IF NOT EXISTS idx_info_items_published_at ON info_items(published_at DESC)")
+        await db.execute("CREATE INDEX IF NOT EXISTS idx_info_items_source ON info_items(source)")
+        await db.commit()
+
+
+async def upsert_items(items: list[dict]) -> int:
+    if not items:
+        return 0
+
+    inserted = 0
+    now = datetime.now(timezone.utc).isoformat()
+
+    async with aiosqlite.connect(DB_PATH) as db:
+        for item in items:
+            cursor = await db.execute(
+                """
+                INSERT OR IGNORE INTO info_items
+                (source, title, summary, url, tags, published_at, collected_at, content_text, content_html)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    item["source"],
+                    item["title"],
+                    item.get("summary", ""),
+                    item["url"],
+                    json.dumps(item.get("tags", []), ensure_ascii=False),
+                    item.get("published_at"),
+                    now,
+                    item.get("content_text"),
+                    item.get("content_html"),
+                ),
+            )
+            inserted += cursor.rowcount
+        await db.commit()
+
+    return inserted

info-engine/docker-compose.yml

@@ -0,0 +1,16 @@
+services:
+  info-engine:
+    build: .
+    container_name: info-engine
+    restart: unless-stopped
+    volumes:
+      - /volume1/docker/info-engine/data:/app/data
+    environment:
+      - DB_PATH=/app/data/info_engine.db
+      - INFO_ENGINE_INTERVAL_SECONDS=${INFO_ENGINE_INTERVAL_SECONDS:-900}
+      - INFO_ENGINE_SOURCES=${INFO_ENGINE_SOURCES:-https://hnrss.org/frontpage,https://lobste.rs/rss}
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"

info-engine/main.py

@@ -0,0 +1,37 @@
+import asyncio
+import logging
+import os
+
+from collector import collect_sources
+from db import init_db, upsert_items
+from processor import process_items
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+
+COLLECT_INTERVAL_SECONDS = int(os.environ.get("INFO_ENGINE_INTERVAL_SECONDS", "900"))
+
+
+async def run_once() -> int:
+    raw_items = await collect_sources()
+    processed = process_items(raw_items)
+    inserted = await upsert_items(processed)
+    return inserted
+
+
+async def main():
+    await init_db()
+    log.info("Info engine started (interval=%ss)", COLLECT_INTERVAL_SECONDS)
+
+    while True:
+        try:
+            inserted = await run_once()
+            log.info("Info engine cycle complete (inserted=%d)", inserted)
+        except Exception as exc:
+            log.error("Info engine cycle failed: %s", exc)
+
+        await asyncio.sleep(COLLECT_INTERVAL_SECONDS)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

info-engine/processor.py

@@ -0,0 +1,45 @@
+from urllib.parse import urldefrag, urlparse
+
+
+def _normalize_url(url: str) -> str:
+    clean, _ = urldefrag(url.strip())
+    parsed = urlparse(clean)
+    if parsed.scheme and parsed.netloc:
+        return clean
+    return ""
+
+
+def process_items(items: list[dict]) -> list[dict]:
+    seen = set()
+    processed = []
+
+    for item in items:
+        title = (item.get("title") or "").strip()
+        url = _normalize_url(item.get("url") or "")
+        if not title or not url:
+            continue
+
+        dedupe_key = url.lower()
+        if dedupe_key in seen:
+            continue
+        seen.add(dedupe_key)
+
+        summary = (item.get("summary") or "").strip()
+        tags = item.get("tags") or []
+        if not isinstance(tags, list):
+            tags = []
+
+        processed.append(
+            {
+                "source": (item.get("source") or "unknown").strip() or "unknown",
+                "title": title,
+                "summary": summary,
+                "url": url,
+                "tags": [str(t).strip() for t in tags if str(t).strip()],
+                "published_at": item.get("published_at"),
+                "content_text": item.get("content_text") or summary,
+                "content_html": item.get("content_html") or summary,
+            }
+        )
+
+    return processed

info-engine/requirements.txt

@@ -0,0 +1,2 @@
+aiosqlite
+httpx