diff --git a/claude-code-tracker/.env.example b/claude-code-tracker/.env.example new file mode 100644 index 0000000..11244ee --- /dev/null +++ b/claude-code-tracker/.env.example @@ -0,0 +1,6 @@ +ANTHROPIC_API_KEY=your_api_key_here +ANTHROPIC_BASE_URL=https://www.bytecatcode.org +SUMMARY_HOUR=23 +COLLECT_INTERVAL=60 +DB_PATH=/app/data/conversations.db +CONVERSATIONS_PATH=/app/conversations diff --git a/claude-code-tracker/Dockerfile b/claude-code-tracker/Dockerfile new file mode 100644 index 0000000..d444f0e --- /dev/null +++ b/claude-code-tracker/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-slim + +WORKDIR /app + +# Copy requirements and install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY *.py ./ + +# Create data directory +RUN mkdir -p /app/data + +# Run as non-root user +RUN useradd -m -u 1000 appuser && chown -R appuser:appuser /app +USER appuser + +CMD ["python", "main.py"] diff --git a/claude-code-tracker/collector.py b/claude-code-tracker/collector.py new file mode 100644 index 0000000..b0705d4 --- /dev/null +++ b/claude-code-tracker/collector.py @@ -0,0 +1,126 @@ +import os +import logging +from pathlib import Path +from datetime import datetime +import asyncio + +import db +import parser + +logger = logging.getLogger(__name__) + +CONVERSATIONS_PATH = os.environ.get("CONVERSATIONS_PATH", "/app/conversations") +COLLECT_INTERVAL = int(os.environ.get("COLLECT_INTERVAL", "60")) + + +async def discover_jsonl_files(): + """Discover all .jsonl files in conversations directory""" + conversations_dir = Path(CONVERSATIONS_PATH) + if not conversations_dir.exists(): + logger.warning(f"Conversations directory does not exist: {CONVERSATIONS_PATH}") + return [] + + jsonl_files = [] + for file_path in conversations_dir.rglob("*.jsonl"): + if file_path.is_file(): + jsonl_files.append(str(file_path)) + + return jsonl_files + + +async def process_file(file_path: str): + """Process a single conversation file incrementally""" + try: + # Get checkpoint + checkpoint = await db.get_checkpoint(file_path) + start_line = checkpoint['last_line_count'] if checkpoint else 0 + + # Get file modification time + file_stat = os.stat(file_path) + file_mtime = datetime.fromtimestamp(file_stat.st_mtime).isoformat() + + # Check if file has been modified + if checkpoint and checkpoint['last_modified'] == file_mtime: + # File hasn't changed, skip + return 0 + + # Count total lines + with open(file_path, 'r') as f: + total_lines = sum(1 for _ in f) + + if total_lines <= start_line: + # No new lines + return 0 + + # Parse new content + logger.info(f"Processing {file_path} from line {start_line} to {total_lines}") + conversation_data = parser.parse_conversation_file(file_path, start_line) + + if not conversation_data: + logger.warning(f"No data extracted from {file_path}") + return 0 + + # Store in database + await db.upsert_conversation( + conversation_data['session_id'], + { + 'project_path': conversation_data['project_path'], + 'git_branch': conversation_data.get('git_branch'), + 'started_at': conversation_data['started_at'], + 'last_updated_at': conversation_data['last_updated_at'], + 'message_count': conversation_data['message_count'], + 'user_message_count': conversation_data['user_message_count'], + 'assistant_message_count': conversation_data['assistant_message_count'], + 'total_input_tokens': conversation_data['total_input_tokens'], + 'total_output_tokens': conversation_data['total_output_tokens'], + 'model': conversation_data.get('model'), + 'slug': conversation_data.get('slug'), + 'file_path': file_path + } + ) + + # Store messages + for msg in conversation_data['messages']: + await db.insert_message(msg) + + # Store tool calls + for tool_call in conversation_data['tool_calls']: + await db.insert_tool_call(tool_call) + + # Update checkpoint + await db.update_checkpoint(file_path, total_lines, file_mtime) + + logger.info(f"Processed {len(conversation_data['messages'])} messages from {file_path}") + return len(conversation_data['messages']) + + except Exception as e: + logger.error(f"Error processing {file_path}: {e}", exc_info=True) + return 0 + + +async def collect_conversations(): + """Main collection loop""" + while True: + try: + logger.info("Starting conversation collection cycle") + + # Discover files + files = await discover_jsonl_files() + logger.info(f"Found {len(files)} conversation files") + + # Process each file + total_messages = 0 + for file_path in files: + messages_processed = await process_file(file_path) + total_messages += messages_processed + + if total_messages > 0: + logger.info(f"Collection cycle complete: processed {total_messages} new messages") + else: + logger.debug("Collection cycle complete: no new messages") + + except Exception as e: + logger.error(f"Error in collection cycle: {e}", exc_info=True) + + # Wait before next cycle + await asyncio.sleep(COLLECT_INTERVAL) diff --git a/claude-code-tracker/db.py b/claude-code-tracker/db.py new file mode 100644 index 0000000..2ee5cbc --- /dev/null +++ b/claude-code-tracker/db.py @@ -0,0 +1,215 @@ +import os +import aiosqlite + +DB_PATH = os.environ.get("DB_PATH", "/app/data/conversations.db") + + +async def init_db(): + """Initialize database with schema""" + async with aiosqlite.connect(DB_PATH) as db: + # Conversations table + await db.execute(""" + CREATE TABLE IF NOT EXISTS conversations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT UNIQUE NOT NULL, + project_path TEXT NOT NULL, + git_branch TEXT, + started_at DATETIME NOT NULL, + last_updated_at DATETIME NOT NULL, + message_count INTEGER DEFAULT 0, + user_message_count INTEGER DEFAULT 0, + assistant_message_count INTEGER DEFAULT 0, + total_input_tokens INTEGER DEFAULT 0, + total_output_tokens INTEGER DEFAULT 0, + model TEXT, + slug TEXT, + file_path TEXT NOT NULL + ) + """) + + # Messages table + await db.execute(""" + CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + message_uuid TEXT UNIQUE NOT NULL, + parent_uuid TEXT, + role TEXT NOT NULL, + content_text TEXT, + timestamp DATETIME NOT NULL, + prompt_id TEXT, + model TEXT, + input_tokens INTEGER, + output_tokens INTEGER, + has_tool_use BOOLEAN DEFAULT 0, + has_thinking BOOLEAN DEFAULT 0, + stop_reason TEXT + ) + """) + + # Tool calls table + await db.execute(""" + CREATE TABLE IF NOT EXISTS tool_calls ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_uuid TEXT NOT NULL, + tool_use_id TEXT UNIQUE NOT NULL, + tool_name TEXT NOT NULL, + tool_input TEXT, + tool_result TEXT, + is_error BOOLEAN DEFAULT 0, + timestamp DATETIME NOT NULL + ) + """) + + # Daily summaries table + await db.execute(""" + CREATE TABLE IF NOT EXISTS daily_summaries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + date TEXT UNIQUE NOT NULL, + project_path TEXT, + conversation_count INTEGER, + total_messages INTEGER, + total_tokens INTEGER, + summary_content TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + """) + + # Checkpoints table + await db.execute(""" + CREATE TABLE IF NOT EXISTS checkpoints ( + file_path TEXT PRIMARY KEY, + last_line_count INTEGER NOT NULL, + last_modified DATETIME NOT NULL + ) + """) + + # Create indexes + await db.execute("CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_messages_timestamp ON messages(timestamp)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_tool_calls_message ON tool_calls(message_uuid)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_conversations_project ON conversations(project_path)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_conversations_started ON conversations(started_at)") + + await db.commit() + + +async def upsert_conversation(session_id, data): + """Insert or update conversation""" + async with aiosqlite.connect(DB_PATH) as db: + await db.execute(""" + INSERT INTO conversations ( + session_id, project_path, git_branch, started_at, last_updated_at, + message_count, user_message_count, assistant_message_count, + total_input_tokens, total_output_tokens, model, slug, file_path + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + last_updated_at = excluded.last_updated_at, + message_count = excluded.message_count, + user_message_count = excluded.user_message_count, + assistant_message_count = excluded.assistant_message_count, + total_input_tokens = excluded.total_input_tokens, + total_output_tokens = excluded.total_output_tokens, + model = excluded.model, + slug = excluded.slug, + git_branch = excluded.git_branch + """, ( + session_id, data['project_path'], data.get('git_branch'), + data['started_at'], data['last_updated_at'], + data['message_count'], data['user_message_count'], data['assistant_message_count'], + data['total_input_tokens'], data['total_output_tokens'], + data.get('model'), data.get('slug'), data['file_path'] + )) + await db.commit() + + +async def insert_message(message_data): + """Insert message (skip if exists)""" + async with aiosqlite.connect(DB_PATH) as db: + try: + await db.execute(""" + INSERT INTO messages ( + session_id, message_uuid, parent_uuid, role, content_text, + timestamp, prompt_id, model, input_tokens, output_tokens, + has_tool_use, has_thinking, stop_reason + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, ( + message_data['session_id'], message_data['message_uuid'], + message_data.get('parent_uuid'), message_data['role'], + message_data.get('content_text'), message_data['timestamp'], + message_data.get('prompt_id'), message_data.get('model'), + message_data.get('input_tokens'), message_data.get('output_tokens'), + message_data.get('has_tool_use', False), message_data.get('has_thinking', False), + message_data.get('stop_reason') + )) + await db.commit() + except aiosqlite.IntegrityError: + # Message already exists, skip + pass + + +async def insert_tool_call(tool_data): + """Insert tool call (skip if exists)""" + async with aiosqlite.connect(DB_PATH) as db: + try: + await db.execute(""" + INSERT INTO tool_calls ( + message_uuid, tool_use_id, tool_name, tool_input, + tool_result, is_error, timestamp + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, ( + tool_data['message_uuid'], tool_data['tool_use_id'], + tool_data['tool_name'], tool_data.get('tool_input'), + tool_data.get('tool_result'), tool_data.get('is_error', False), + tool_data['timestamp'] + )) + await db.commit() + except aiosqlite.IntegrityError: + # Tool call already exists, skip + pass + + +async def get_checkpoint(file_path): + """Get checkpoint for a file""" + async with aiosqlite.connect(DB_PATH) as db: + cursor = await db.execute( + "SELECT last_line_count, last_modified FROM checkpoints WHERE file_path = ?", + (file_path,) + ) + row = await cursor.fetchone() + return {"last_line_count": row[0], "last_modified": row[1]} if row else None + + +async def update_checkpoint(file_path, line_count, modified_time): + """Update checkpoint for a file""" + async with aiosqlite.connect(DB_PATH) as db: + await db.execute(""" + INSERT INTO checkpoints (file_path, last_line_count, last_modified) + VALUES (?, ?, ?) + ON CONFLICT(file_path) DO UPDATE SET + last_line_count = excluded.last_line_count, + last_modified = excluded.last_modified + """, (file_path, line_count, modified_time)) + await db.commit() + + +async def upsert_summary(date, data): + """Insert or update daily summary""" + async with aiosqlite.connect(DB_PATH) as db: + await db.execute(""" + INSERT INTO daily_summaries ( + date, project_path, conversation_count, total_messages, + total_tokens, summary_content + ) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(date) DO UPDATE SET + project_path = excluded.project_path, + conversation_count = excluded.conversation_count, + total_messages = excluded.total_messages, + total_tokens = excluded.total_tokens, + summary_content = excluded.summary_content, + created_at = CURRENT_TIMESTAMP + """, ( + date, data.get('project_path'), data['conversation_count'], + data['total_messages'], data['total_tokens'], data['summary_content'] + )) + await db.commit() diff --git a/claude-code-tracker/docker-compose.yml b/claude-code-tracker/docker-compose.yml new file mode 100644 index 0000000..cc57a14 --- /dev/null +++ b/claude-code-tracker/docker-compose.yml @@ -0,0 +1,21 @@ +services: + claude-code-tracker: + build: . + container_name: claude-code-tracker + restart: unless-stopped + volumes: + - /volume1/docker/claude-code-tracker/data:/app/data + - /volume1/docker/claude-code-tracker/conversations:/app/conversations:ro + environment: + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL:-https://www.bytecatcode.org} + - SUMMARY_HOUR=${SUMMARY_HOUR:-23} + - COLLECT_INTERVAL=${COLLECT_INTERVAL:-60} + - DB_PATH=/app/data/conversations.db + - CONVERSATIONS_PATH=/app/conversations + - TRIGGER_PATH=/app/data/trigger + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" diff --git a/claude-code-tracker/main.py b/claude-code-tracker/main.py new file mode 100644 index 0000000..c203fcd --- /dev/null +++ b/claude-code-tracker/main.py @@ -0,0 +1,47 @@ +import os +import logging +import asyncio + +import db +import collector +import summarizer + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + + +async def main(): + """Main service loop""" + logger.info("Starting Claude Code Tracker service") + + # Initialize database + await db.init_db() + logger.info("Database initialized") + + # Create data directory if it doesn't exist + data_dir = os.path.dirname(db.DB_PATH) + os.makedirs(data_dir, exist_ok=True) + + # Start collection and summarization tasks + tasks = [ + asyncio.create_task(collector.collect_conversations()), + asyncio.create_task(summarizer.summarize_daily()) + ] + + logger.info("Service started - collection and summarization tasks running") + + try: + await asyncio.gather(*tasks) + except KeyboardInterrupt: + logger.info("Shutting down...") + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/claude-code-tracker/parser.py b/claude-code-tracker/parser.py new file mode 100644 index 0000000..16d8efd --- /dev/null +++ b/claude-code-tracker/parser.py @@ -0,0 +1,238 @@ +import json +import logging +from datetime import datetime +from typing import Dict, List, Any, Optional + +logger = logging.getLogger(__name__) + + +def parse_jsonl_file(file_path: str, start_line: int = 0) -> List[Dict[str, Any]]: + """Parse JSONL file from a specific line number""" + messages = [] + try: + with open(file_path, 'r', encoding='utf-8') as f: + for i, line in enumerate(f): + if i < start_line: + continue + if not line.strip(): + continue + try: + obj = json.loads(line) + messages.append(obj) + except json.JSONDecodeError as e: + logger.warning(f"Failed to parse line {i} in {file_path}: {e}") + continue + except Exception as e: + logger.error(f"Failed to read {file_path}: {e}") + return messages + + +def extract_text_content(content: Any) -> str: + """Extract text from message content""" + if isinstance(content, str): + return content + + if isinstance(content, list): + text_parts = [] + for block in content: + if isinstance(block, dict): + if block.get('type') == 'text': + text_parts.append(block.get('text', '')) + elif block.get('type') == 'tool_result': + # Include tool results in searchable text + result = block.get('content', '') + if isinstance(result, str): + text_parts.append(result[:500]) # Limit length + return '\n'.join(text_parts) + + return '' + + +def extract_tool_calls(content: Any, message_uuid: str, timestamp: str) -> List[Dict[str, Any]]: + """Extract tool use blocks from message content""" + tool_calls = [] + + if not isinstance(content, list): + return tool_calls + + for block in content: + if isinstance(block, dict) and block.get('type') == 'tool_use': + tool_calls.append({ + 'message_uuid': message_uuid, + 'tool_use_id': block.get('id'), + 'tool_name': block.get('name'), + 'tool_input': json.dumps(block.get('input', {})), + 'tool_result': None, + 'is_error': False, + 'timestamp': timestamp + }) + + return tool_calls + + +def extract_tool_results(content: Any, tool_calls_map: Dict[str, Dict]) -> None: + """Extract tool results and match them to tool calls""" + if not isinstance(content, list): + return + + for block in content: + if isinstance(block, dict) and block.get('type') == 'tool_result': + tool_use_id = block.get('tool_use_id') + if tool_use_id and tool_use_id in tool_calls_map: + result_content = block.get('content', '') + if isinstance(result_content, str): + tool_calls_map[tool_use_id]['tool_result'] = result_content[:5000] # Limit length + tool_calls_map[tool_use_id]['is_error'] = block.get('is_error', False) + + +def extract_message_data(obj: Dict[str, Any], session_id: str) -> Optional[Dict[str, Any]]: + """Extract relevant fields from a message object""" + msg_type = obj.get('type') + + # Only process user and assistant messages + if msg_type not in ['user', 'assistant']: + return None + + message = obj.get('message', {}) + role = message.get('role') + + if not role or role not in ['user', 'assistant']: + return None + + content = message.get('content', '') + has_tool_use = False + has_thinking = False + + # Check for tool use and thinking blocks + if isinstance(content, list): + for block in content: + if isinstance(block, dict): + if block.get('type') == 'tool_use': + has_tool_use = True + elif block.get('type') == 'thinking': + has_thinking = True + + # Extract usage metrics + usage = message.get('usage', {}) + input_tokens = usage.get('input_tokens', 0) + output_tokens = usage.get('output_tokens', 0) + + return { + 'session_id': session_id, + 'message_uuid': obj.get('uuid'), + 'parent_uuid': obj.get('parentUuid'), + 'role': role, + 'content_text': extract_text_content(content), + 'timestamp': obj.get('timestamp'), + 'prompt_id': obj.get('promptId'), + 'model': message.get('model'), + 'input_tokens': input_tokens, + 'output_tokens': output_tokens, + 'has_tool_use': has_tool_use, + 'has_thinking': has_thinking, + 'stop_reason': message.get('stop_reason') + } + + +def calculate_conversation_stats(messages: List[Dict[str, Any]]) -> Dict[str, Any]: + """Calculate aggregate statistics for a conversation""" + stats = { + 'message_count': 0, + 'user_message_count': 0, + 'assistant_message_count': 0, + 'total_input_tokens': 0, + 'total_output_tokens': 0, + 'started_at': None, + 'last_updated_at': None, + 'model': None + } + + for msg in messages: + if msg.get('role') == 'user': + stats['user_message_count'] += 1 + elif msg.get('role') == 'assistant': + stats['assistant_message_count'] += 1 + if not stats['model'] and msg.get('model'): + stats['model'] = msg.get('model') + + stats['message_count'] += 1 + stats['total_input_tokens'] += msg.get('input_tokens', 0) + stats['total_output_tokens'] += msg.get('output_tokens', 0) + + timestamp = msg.get('timestamp') + if timestamp: + if not stats['started_at'] or timestamp < stats['started_at']: + stats['started_at'] = timestamp + if not stats['last_updated_at'] or timestamp > stats['last_updated_at']: + stats['last_updated_at'] = timestamp + + return stats + + +def parse_conversation_file(file_path: str, start_line: int = 0) -> Dict[str, Any]: + """Parse a conversation file and extract all data""" + raw_messages = parse_jsonl_file(file_path, start_line) + + if not raw_messages: + return None + + # Extract session metadata from first message + session_id = None + project_path = None + git_branch = None + slug = None + + for obj in raw_messages: + if obj.get('sessionId'): + session_id = obj['sessionId'] + if obj.get('cwd'): + project_path = obj['cwd'] + if obj.get('gitBranch'): + git_branch = obj['gitBranch'] + if obj.get('slug'): + slug = obj['slug'] + + if session_id and project_path: + break + + if not session_id: + logger.warning(f"No session ID found in {file_path}") + return None + + # Extract messages + messages = [] + tool_calls_map = {} # Map tool_use_id to tool call data + + for obj in raw_messages: + msg_data = extract_message_data(obj, session_id) + if msg_data: + messages.append(msg_data) + + # Extract tool calls from assistant messages + if msg_data['role'] == 'assistant' and msg_data['has_tool_use']: + message_content = obj.get('message', {}).get('content', []) + tool_calls = extract_tool_calls(message_content, msg_data['message_uuid'], msg_data['timestamp']) + for tc in tool_calls: + tool_calls_map[tc['tool_use_id']] = tc + + # Extract tool results from user messages + if obj.get('type') == 'user': + message_content = obj.get('message', {}).get('content', []) + extract_tool_results(message_content, tool_calls_map) + + if not messages: + return None + + # Calculate stats + stats = calculate_conversation_stats(messages) + + return { + 'session_id': session_id, + 'project_path': project_path or 'unknown', + 'git_branch': git_branch, + 'slug': slug, + 'file_path': file_path, + 'messages': messages, + 'tool_calls': list(tool_calls_map.values()), + **stats + } diff --git a/claude-code-tracker/requirements.txt b/claude-code-tracker/requirements.txt new file mode 100644 index 0000000..28d85f8 --- /dev/null +++ b/claude-code-tracker/requirements.txt @@ -0,0 +1,3 @@ +aiosqlite==0.19.0 +anthropic==0.39.0 +httpx==0.27.0 diff --git a/claude-code-tracker/summarizer.py b/claude-code-tracker/summarizer.py new file mode 100644 index 0000000..43a4527 --- /dev/null +++ b/claude-code-tracker/summarizer.py @@ -0,0 +1,174 @@ +import os +import logging +from datetime import datetime, timedelta +import asyncio +import aiosqlite +from anthropic import AsyncAnthropic + +import db + +logger = logging.getLogger(__name__) + +ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY") +ANTHROPIC_BASE_URL = os.environ.get("ANTHROPIC_BASE_URL", "https://www.bytecatcode.org") +SUMMARY_HOUR = int(os.environ.get("SUMMARY_HOUR", "23")) +TRIGGER_PATH = os.environ.get("TRIGGER_PATH", "/app/data/trigger") + + +async def generate_summary_for_date(date_str: str): + """Generate summary for a specific date""" + try: + logger.info(f"Generating summary for {date_str}") + + # Query conversations for this date + async with aiosqlite.connect(db.DB_PATH) as conn: + # Get conversations + cursor = await conn.execute(""" + SELECT session_id, project_path, slug, message_count, + total_input_tokens, total_output_tokens + FROM conversations + WHERE DATE(started_at) = ? OR DATE(last_updated_at) = ? + ORDER BY started_at + """, (date_str, date_str)) + conversations = await cursor.fetchall() + + if not conversations: + logger.info(f"No conversations found for {date_str}") + return + + # Get sample messages from each conversation + conversation_summaries = [] + total_messages = 0 + total_tokens = 0 + projects = set() + + for conv in conversations: + session_id, project_path, slug, msg_count, input_tokens, output_tokens = conv + projects.add(project_path) + total_messages += msg_count + total_tokens += (input_tokens + output_tokens) + + # Get first few user messages to understand what was worked on + cursor = await conn.execute(""" + SELECT content_text FROM messages + WHERE session_id = ? AND role = 'user' AND content_text IS NOT NULL + ORDER BY timestamp + LIMIT 3 + """, (session_id,)) + user_messages = await cursor.fetchall() + + # Get tool usage + cursor = await conn.execute(""" + SELECT tool_name, COUNT(*) as count + FROM tool_calls + WHERE message_uuid IN ( + SELECT message_uuid FROM messages WHERE session_id = ? + ) + GROUP BY tool_name + ORDER BY count DESC + LIMIT 5 + """, (session_id,)) + tools = await cursor.fetchall() + + conversation_summaries.append({ + 'slug': slug or 'Untitled', + 'project': project_path.split('/')[-1] if project_path else 'unknown', + 'messages': msg_count, + 'tokens': input_tokens + output_tokens, + 'user_messages': [msg[0][:200] for msg in user_messages if msg[0]], + 'tools': [f"{tool[0]} ({tool[1]}x)" for tool in tools] + }) + + # Format for Claude + formatted_convs = [] + for i, conv in enumerate(conversation_summaries, 1): + formatted_convs.append(f""" +**Conversation {i}: {conv['slug']}** +- Project: {conv['project']} +- Messages: {conv['messages']} ({conv['tokens']} tokens) +- Initial requests: {', '.join(conv['user_messages'][:2]) if conv['user_messages'] else 'N/A'} +- Tools used: {', '.join(conv['tools']) if conv['tools'] else 'None'} +""") + + prompt = f"""You are analyzing a developer's Claude Code conversations from {date_str}. + +**Overview:** +- Conversations: {len(conversations)} +- Total messages: {total_messages} +- Total tokens: {total_tokens:,} +- Projects: {', '.join(sorted(projects))} + +**Conversations:** +{''.join(formatted_convs)} + +Generate a concise daily summary covering: +1. Main tasks and goals worked on +2. Key decisions and solutions implemented +3. Files and components modified (if evident from tool usage) +4. Challenges encountered and how they were resolved +5. Tools and patterns used +6. Overall progress and outcomes + +Use markdown formatting. Be specific and technical. Focus on what was accomplished. Keep it under 500 words.""" + + # Call Claude API + client = AsyncAnthropic(api_key=ANTHROPIC_API_KEY, base_url=ANTHROPIC_BASE_URL) + response = await client.messages.create( + model="claude-haiku-4-5-20251001", + max_tokens=4096, + messages=[{"role": "user", "content": prompt}] + ) + + summary_content = response.content[0].text + + # Store summary + await db.upsert_summary(date_str, { + 'project_path': ', '.join(sorted(projects)), + 'conversation_count': len(conversations), + 'total_messages': total_messages, + 'total_tokens': total_tokens, + 'summary_content': summary_content + }) + + logger.info(f"Summary generated for {date_str}") + + except Exception as e: + logger.error(f"Error generating summary for {date_str}: {e}", exc_info=True) + + +async def check_trigger(): + """Check for manual trigger file""" + if os.path.exists(TRIGGER_PATH): + try: + os.remove(TRIGGER_PATH) + logger.info("Manual trigger detected") + # Generate summary for yesterday + yesterday = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d") + await generate_summary_for_date(yesterday) + except Exception as e: + logger.error(f"Error processing trigger: {e}") + + +async def summarize_daily(): + """Daily summarization loop""" + while True: + try: + now = datetime.now() + + # Check for manual trigger + await check_trigger() + + # Check if it's time for daily summary + if now.hour == SUMMARY_HOUR and now.minute < 5: + yesterday = (now - timedelta(days=1)).strftime("%Y-%m-%d") + await generate_summary_for_date(yesterday) + + # Sleep until next hour to avoid duplicate runs + await asyncio.sleep(3600) + else: + # Check every 5 minutes + await asyncio.sleep(300) + + except Exception as e: + logger.error(f"Error in summarization loop: {e}", exc_info=True) + await asyncio.sleep(300) diff --git a/dashboard/backend/config.py b/dashboard/backend/config.py index fda104c..c038243 100644 --- a/dashboard/backend/config.py +++ b/dashboard/backend/config.py @@ -54,6 +54,9 @@ CORS_ORIGINS = os.environ.get("CORS_ORIGINS", "https://nas.jimmygan.com,https:// # Chat Summary CHAT_SUMMARY_DB = os.environ.get("CHAT_SUMMARY_DB", "/app/data/chat-summarizer/chat_summary.db") +# Conversation Tracker +CONVERSATION_TRACKER_DB = os.environ.get("CONVERSATION_TRACKER_DB", "/app/data/claude-code-tracker/conversations.db") + # Info Engine INFO_ENGINE_DB = os.environ.get("INFO_ENGINE_DB", "/app/data/info-engine/info_engine.db") diff --git a/dashboard/backend/main.py b/dashboard/backend/main.py index 28b2a88..4888eb0 100644 --- a/dashboard/backend/main.py +++ b/dashboard/backend/main.py @@ -25,6 +25,7 @@ from routers import auth as auth_router from routers import ( cc_connect, chat_summary, + conversation_tracker, docker_router, files, gitea, @@ -296,6 +297,10 @@ app.include_router( prefix="/api/chat-summary", dependencies=[Depends(_inject_user), Depends(require_page("chat-digest"))], ) +app.include_router( + conversation_tracker.router, + dependencies=[Depends(_inject_user), Depends(require_page("conversations"))], +) app.include_router( info_engine.router, prefix="/api/info-engine", diff --git a/dashboard/backend/routers/conversation_tracker.py b/dashboard/backend/routers/conversation_tracker.py new file mode 100644 index 0000000..e1012e5 --- /dev/null +++ b/dashboard/backend/routers/conversation_tracker.py @@ -0,0 +1,286 @@ +import os +import aiosqlite +from fastapi import APIRouter, HTTPException, Query +from typing import Optional + +router = APIRouter(prefix="/api/conversations", tags=["conversations"]) + +CONVERSATION_TRACKER_DB = os.environ.get("CONVERSATION_TRACKER_DB", "/app/data/claude-code-tracker/conversations.db") + + +async def _db(): + return aiosqlite.connect(CONVERSATION_TRACKER_DB) + + +@router.get("/dates") +async def list_dates(): + """List all dates with conversations""" + async with await _db() as db: + cursor = await db.execute(""" + SELECT DISTINCT DATE(started_at) as date + FROM conversations + ORDER BY date DESC + """) + return [r[0] for r in await cursor.fetchall()] + + +@router.get("/summary/{date}") +async def get_summary(date: str): + """Get daily summary for a date""" + async with await _db() as db: + cursor = await db.execute(""" + SELECT summary_content, conversation_count, total_messages, + total_tokens, created_at, project_path + FROM daily_summaries + WHERE date = ? + """, (date,)) + row = await cursor.fetchone() + + if not row: + raise HTTPException(404, "No summary for this date") + + return { + "date": date, + "content": row[0], + "conversation_count": row[1], + "total_messages": row[2], + "total_tokens": row[3], + "created_at": row[4], + "projects": row[5] + } + + +@router.get("/list") +async def list_conversations( + date: Optional[str] = None, + project_path: Optional[str] = None, + limit: int = Query(50, le=200), + offset: int = 0 +): + """List conversations with filters""" + async with await _db() as db: + query = "SELECT session_id, project_path, slug, started_at, message_count, total_input_tokens, total_output_tokens, model FROM conversations WHERE 1=1" + params = [] + + if date: + query += " AND DATE(started_at) = ?" + params.append(date) + + if project_path: + query += " AND project_path LIKE ?" + params.append(f"%{project_path}%") + + query += " ORDER BY started_at DESC LIMIT ? OFFSET ?" + params.extend([limit, offset]) + + cursor = await db.execute(query, params) + rows = await cursor.fetchall() + + return [{ + "session_id": r[0], + "project_path": r[1], + "slug": r[2], + "started_at": r[3], + "message_count": r[4], + "total_tokens": r[5] + r[6], + "model": r[7] + } for r in rows] + + +@router.get("/{session_id}") +async def get_conversation(session_id: str): + """Get conversation details""" + async with await _db() as db: + # Get conversation metadata + cursor = await db.execute(""" + SELECT project_path, git_branch, slug, started_at, last_updated_at, + message_count, total_input_tokens, total_output_tokens, model + FROM conversations + WHERE session_id = ? + """, (session_id,)) + conv = await cursor.fetchone() + + if not conv: + raise HTTPException(404, "Conversation not found") + + # Get message count + cursor = await db.execute(""" + SELECT COUNT(*) FROM messages WHERE session_id = ? + """, (session_id,)) + msg_count = (await cursor.fetchone())[0] + + return { + "session_id": session_id, + "project_path": conv[0], + "git_branch": conv[1], + "slug": conv[2], + "started_at": conv[3], + "last_updated_at": conv[4], + "message_count": msg_count, + "total_input_tokens": conv[6], + "total_output_tokens": conv[7], + "model": conv[8] + } + + +@router.get("/{session_id}/messages") +async def get_messages( + session_id: str, + limit: int = Query(100, le=500), + offset: int = 0 +): + """Get messages for a conversation""" + async with await _db() as db: + cursor = await db.execute(""" + SELECT message_uuid, role, content_text, timestamp, model, + input_tokens, output_tokens, has_tool_use, has_thinking + FROM messages + WHERE session_id = ? + ORDER BY timestamp + LIMIT ? OFFSET ? + """, (session_id, limit, offset)) + rows = await cursor.fetchall() + + messages = [] + for r in rows: + msg = { + "uuid": r[0], + "role": r[1], + "content": r[2], + "timestamp": r[3], + "model": r[4], + "input_tokens": r[5], + "output_tokens": r[6], + "has_tool_use": bool(r[7]), + "has_thinking": bool(r[8]), + "tool_calls": [] + } + + # Get tool calls for this message + if msg["has_tool_use"]: + cursor2 = await db.execute(""" + SELECT tool_name, tool_input, tool_result, is_error + FROM tool_calls + WHERE message_uuid = ? + """, (r[0],)) + tool_rows = await cursor2.fetchall() + msg["tool_calls"] = [{ + "name": tr[0], + "input": tr[1], + "result": tr[2], + "is_error": bool(tr[3]) + } for tr in tool_rows] + + messages.append(msg) + + return messages + + +@router.get("/search") +async def search_conversations( + q: str = Query(..., min_length=2), + date_from: Optional[str] = None, + date_to: Optional[str] = None, + limit: int = Query(50, le=200) +): + """Search conversations by text""" + async with await _db() as db: + query = """ + SELECT DISTINCT m.session_id, c.project_path, c.slug, c.started_at, + c.message_count, c.total_input_tokens, c.total_output_tokens + FROM messages m + JOIN conversations c ON m.session_id = c.session_id + WHERE m.content_text LIKE ? + """ + params = [f"%{q}%"] + + if date_from: + query += " AND DATE(c.started_at) >= ?" + params.append(date_from) + + if date_to: + query += " AND DATE(c.started_at) <= ?" + params.append(date_to) + + query += " ORDER BY c.started_at DESC LIMIT ?" + params.append(limit) + + cursor = await db.execute(query, params) + rows = await cursor.fetchall() + + return [{ + "session_id": r[0], + "project_path": r[1], + "slug": r[2], + "started_at": r[3], + "message_count": r[4], + "total_tokens": r[5] + r[6] + } for r in rows] + + +@router.get("/stats") +async def get_stats( + date_from: Optional[str] = None, + date_to: Optional[str] = None +): + """Get overall statistics""" + async with await _db() as db: + query = "SELECT COUNT(*), SUM(message_count), SUM(total_input_tokens + total_output_tokens) FROM conversations WHERE 1=1" + params = [] + + if date_from: + query += " AND DATE(started_at) >= ?" + params.append(date_from) + + if date_to: + query += " AND DATE(started_at) <= ?" + params.append(date_to) + + cursor = await db.execute(query, params) + stats = await cursor.fetchone() + + # Get top projects + cursor = await db.execute(""" + SELECT project_path, COUNT(*) as count + FROM conversations + GROUP BY project_path + ORDER BY count DESC + LIMIT 10 + """) + projects = await cursor.fetchall() + + # Get tool usage + cursor = await db.execute(""" + SELECT tool_name, COUNT(*) as count + FROM tool_calls + GROUP BY tool_name + ORDER BY count DESC + LIMIT 10 + """) + tools = await cursor.fetchall() + + return { + "total_conversations": stats[0] or 0, + "total_messages": stats[1] or 0, + "total_tokens": stats[2] or 0, + "top_projects": [{"path": p[0], "count": p[1]} for p in projects], + "top_tools": [{"name": t[0], "count": t[1]} for t in tools] + } + + +@router.post("/trigger") +async def trigger_summary(): + """Trigger manual summary generation""" + trigger_path = os.environ.get("TRIGGER_PATH", "/app/data/claude-code-tracker/trigger") + + # Validate path + trigger_path = os.path.abspath(trigger_path) + forbidden_prefixes = ("/etc/", "/root/", "/sys/", "/proc/", "/boot/") + if any(trigger_path.startswith(prefix) for prefix in forbidden_prefixes): + raise HTTPException(status_code=400, detail="Invalid trigger path") + + os.makedirs(os.path.dirname(trigger_path), exist_ok=True) + with open(trigger_path, "w") as f: + f.write("1") + + return {"status": "triggered"} diff --git a/dashboard/frontend/src/App.svelte b/dashboard/frontend/src/App.svelte index 68eff72..07f55e6 100644 --- a/dashboard/frontend/src/App.svelte +++ b/dashboard/frontend/src/App.svelte @@ -8,6 +8,7 @@ import OpenClaw from "./routes/OpenClaw.svelte"; import Settings from "./routes/Settings.svelte"; import ChatSummary from "./routes/ChatSummary.svelte"; + import Conversations from "./routes/Conversations.svelte"; import Security from "./routes/Security.svelte"; import LiteLLM from "./routes/LiteLLM.svelte"; import CcConnect from "./routes/CcConnect.svelte"; @@ -26,6 +27,7 @@ "terminal", "openclaw", "chat-digest", + "conversations", "settings", "security", "litellm", @@ -201,6 +203,8 @@ {:else if page === "chat-digest" && hasPageAccess("chat-digest")} + {:else if page === "conversations" && hasPageAccess("conversations")} + {:else if page === "settings" && userRole === "admin"} {:else if page === "security" && hasPageAccess("security")} diff --git a/dashboard/frontend/src/components/Sidebar.svelte b/dashboard/frontend/src/components/Sidebar.svelte index 1d62b32..c0a58c1 100644 --- a/dashboard/frontend/src/components/Sidebar.svelte +++ b/dashboard/frontend/src/components/Sidebar.svelte @@ -49,6 +49,7 @@ { id: "openclaw", label: "OpenClaw", icon: "openclaw" }, { id: "cc-connect", label: "cc-connect", icon: "users" }, { id: "chat-digest", label: "Chat Digest", icon: "chat" }, + { id: "conversations", label: "Code Sessions", icon: "code" }, { id: "gitea", label: "Repos", icon: "git" }, { label: "Gitea Web", remoteHref: "https://git.jimmygan.com:8443", icon: "git", external: true }, { label: "Stirling PDF", remoteHref: "https://pdf.jimmygan.com:8443", icon: "pdf", external: true }, @@ -261,6 +262,8 @@ {:else if icon === "chat"} + {:else if icon === "code"} + {:else if icon === "git"} {:else if icon === "pdf"} diff --git a/dashboard/frontend/src/routes/Conversations.svelte b/dashboard/frontend/src/routes/Conversations.svelte new file mode 100644 index 0000000..f9af7df --- /dev/null +++ b/dashboard/frontend/src/routes/Conversations.svelte @@ -0,0 +1,306 @@ + + +
+
+

Code Sessions

+
+ + +
+
+ + {#if error} +
{error}
+ {/if} + + +
+ e.key === "Enter" && search()} + placeholder="Search conversations..." + class="w-full px-4 py-2 text-sm border border-surface-200 dark:border-surface-600 rounded-lg focus:outline-none focus:ring-2 focus:ring-primary-500 dark:bg-surface-700 dark:text-white" + /> +
+ + + {#if view !== "search" && view !== "stats"} +
+ {#each dates as d} + + {/each} + {#if !dates.length && !error} +

No conversations yet.

+ {/if} +
+ {/if} + + {#if loading} +
+ {:else if view === "summary" && summary} + +
+
+
Conversations
+
{summary.conversation_count}
+
+
+
Messages
+
{summary.total_messages}
+
+
+
Tokens
+
{formatTokens(summary.total_tokens)}
+
+
+
Projects
+
{summary.projects?.split(',')[0] || 'N/A'}
+
+
+ +
+ {@html markdownToHtml(summary.content)} +
+ + +
+
+ Conversations ({conversations.length}) +
+ {#each conversations as conv} + + {/each} +
+ + {:else if view === "detail" && selectedConversation} + + + +
+

{selectedConversation.slug || 'Untitled'}

+
+
Project: {selectedConversation.project_path?.split('/').pop()}
+
Branch: {selectedConversation.git_branch || 'N/A'}
+
Messages: {selectedConversation.message_count}
+
Tokens: {formatTokens(selectedConversation.total_input_tokens + selectedConversation.total_output_tokens)}
+
+
+ + +
+ {#each messages as msg} +
+
+ {msg.role} + {msg.timestamp?.slice(11, 19)} +
+
{msg.content?.slice(0, 500)}{msg.content?.length > 500 ? '...' : ''}
+ {#if msg.tool_calls?.length} +
+
Tools: {msg.tool_calls.map(t => t.name).join(', ')}
+
+ {/if} + {#if msg.input_tokens || msg.output_tokens} +
+ {msg.input_tokens ? `${msg.input_tokens} in` : ''} {msg.output_tokens ? `${msg.output_tokens} out` : ''} +
+ {/if} +
+ {/each} +
+ + {:else if view === "search"} + + +
+
+ Search Results ({searchResults.length}) +
+ {#each searchResults as conv} + + {/each} + {#if searchResults.length === 0} +
No results found
+ {/if} +
+ + {:else if view === "stats" && stats} + + +
+
+
Total Conversations
+
{stats.total_conversations}
+
+
+
Total Messages
+
{stats.total_messages}
+
+
+
Total Tokens
+
{formatTokens(stats.total_tokens)}
+
+
+ +
+
+

Top Projects

+ {#each stats.top_projects as proj} +
+ {proj.path?.split('/').pop()} + {proj.count} +
+ {/each} +
+ +
+

Top Tools

+ {#each stats.top_tools as tool} +
+ {tool.name} + {tool.count} +
+ {/each} +
+
+ {/if} +