feat: add Claude Code conversation tracker
- Create claude-code-tracker service to monitor and parse Claude Code conversations - Parse JSONL format with messages, tool calls, and metadata - Store in SQLite with full-text search support - Generate daily summaries using Claude API - Add Conversations UI with search, stats, and conversation browsing - Integrate with dashboard backend and frontend - Add sync script for Mac to NAS file transfer
This commit is contained in:
Gan, Jimmy
@@ -0,0 +1,6 @@
|
||||
ANTHROPIC_API_KEY=your_api_key_here
|
||||
ANTHROPIC_BASE_URL=https://www.bytecatcode.org
|
||||
SUMMARY_HOUR=23
|
||||
COLLECT_INTERVAL=60
|
||||
DB_PATH=/app/data/conversations.db
|
||||
CONVERSATIONS_PATH=/app/conversations
|
||||
@@ -0,0 +1,19 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy requirements and install dependencies
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Copy application code
|
||||
COPY *.py ./
|
||||
|
||||
# Create data directory
|
||||
RUN mkdir -p /app/data
|
||||
|
||||
# Run as non-root user
|
||||
RUN useradd -m -u 1000 appuser && chown -R appuser:appuser /app
|
||||
USER appuser
|
||||
|
||||
CMD ["python", "main.py"]
|
||||
@@ -0,0 +1,126 @@
|
||||
import os
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
import asyncio
|
||||
|
||||
import db
|
||||
import parser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CONVERSATIONS_PATH = os.environ.get("CONVERSATIONS_PATH", "/app/conversations")
|
||||
COLLECT_INTERVAL = int(os.environ.get("COLLECT_INTERVAL", "60"))
|
||||
|
||||
|
||||
async def discover_jsonl_files():
|
||||
"""Discover all .jsonl files in conversations directory"""
|
||||
conversations_dir = Path(CONVERSATIONS_PATH)
|
||||
if not conversations_dir.exists():
|
||||
logger.warning(f"Conversations directory does not exist: {CONVERSATIONS_PATH}")
|
||||
return []
|
||||
|
||||
jsonl_files = []
|
||||
for file_path in conversations_dir.rglob("*.jsonl"):
|
||||
if file_path.is_file():
|
||||
jsonl_files.append(str(file_path))
|
||||
|
||||
return jsonl_files
|
||||
|
||||
|
||||
async def process_file(file_path: str):
|
||||
"""Process a single conversation file incrementally"""
|
||||
try:
|
||||
# Get checkpoint
|
||||
checkpoint = await db.get_checkpoint(file_path)
|
||||
start_line = checkpoint['last_line_count'] if checkpoint else 0
|
||||
|
||||
# Get file modification time
|
||||
file_stat = os.stat(file_path)
|
||||
file_mtime = datetime.fromtimestamp(file_stat.st_mtime).isoformat()
|
||||
|
||||
# Check if file has been modified
|
||||
if checkpoint and checkpoint['last_modified'] == file_mtime:
|
||||
# File hasn't changed, skip
|
||||
return 0
|
||||
|
||||
# Count total lines
|
||||
with open(file_path, 'r') as f:
|
||||
total_lines = sum(1 for _ in f)
|
||||
|
||||
if total_lines <= start_line:
|
||||
# No new lines
|
||||
return 0
|
||||
|
||||
# Parse new content
|
||||
logger.info(f"Processing {file_path} from line {start_line} to {total_lines}")
|
||||
conversation_data = parser.parse_conversation_file(file_path, start_line)
|
||||
|
||||
if not conversation_data:
|
||||
logger.warning(f"No data extracted from {file_path}")
|
||||
return 0
|
||||
|
||||
# Store in database
|
||||
await db.upsert_conversation(
|
||||
conversation_data['session_id'],
|
||||
{
|
||||
'project_path': conversation_data['project_path'],
|
||||
'git_branch': conversation_data.get('git_branch'),
|
||||
'started_at': conversation_data['started_at'],
|
||||
'last_updated_at': conversation_data['last_updated_at'],
|
||||
'message_count': conversation_data['message_count'],
|
||||
'user_message_count': conversation_data['user_message_count'],
|
||||
'assistant_message_count': conversation_data['assistant_message_count'],
|
||||
'total_input_tokens': conversation_data['total_input_tokens'],
|
||||
'total_output_tokens': conversation_data['total_output_tokens'],
|
||||
'model': conversation_data.get('model'),
|
||||
'slug': conversation_data.get('slug'),
|
||||
'file_path': file_path
|
||||
}
|
||||
)
|
||||
|
||||
# Store messages
|
||||
for msg in conversation_data['messages']:
|
||||
await db.insert_message(msg)
|
||||
|
||||
# Store tool calls
|
||||
for tool_call in conversation_data['tool_calls']:
|
||||
await db.insert_tool_call(tool_call)
|
||||
|
||||
# Update checkpoint
|
||||
await db.update_checkpoint(file_path, total_lines, file_mtime)
|
||||
|
||||
logger.info(f"Processed {len(conversation_data['messages'])} messages from {file_path}")
|
||||
return len(conversation_data['messages'])
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing {file_path}: {e}", exc_info=True)
|
||||
return 0
|
||||
|
||||
|
||||
async def collect_conversations():
|
||||
"""Main collection loop"""
|
||||
while True:
|
||||
try:
|
||||
logger.info("Starting conversation collection cycle")
|
||||
|
||||
# Discover files
|
||||
files = await discover_jsonl_files()
|
||||
logger.info(f"Found {len(files)} conversation files")
|
||||
|
||||
# Process each file
|
||||
total_messages = 0
|
||||
for file_path in files:
|
||||
messages_processed = await process_file(file_path)
|
||||
total_messages += messages_processed
|
||||
|
||||
if total_messages > 0:
|
||||
logger.info(f"Collection cycle complete: processed {total_messages} new messages")
|
||||
else:
|
||||
logger.debug("Collection cycle complete: no new messages")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in collection cycle: {e}", exc_info=True)
|
||||
|
||||
# Wait before next cycle
|
||||
await asyncio.sleep(COLLECT_INTERVAL)
|
||||
@@ -0,0 +1,215 @@
|
||||
import os
|
||||
import aiosqlite
|
||||
|
||||
DB_PATH = os.environ.get("DB_PATH", "/app/data/conversations.db")
|
||||
|
||||
|
||||
async def init_db():
|
||||
"""Initialize database with schema"""
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
# Conversations table
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS conversations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT UNIQUE NOT NULL,
|
||||
project_path TEXT NOT NULL,
|
||||
git_branch TEXT,
|
||||
started_at DATETIME NOT NULL,
|
||||
last_updated_at DATETIME NOT NULL,
|
||||
message_count INTEGER DEFAULT 0,
|
||||
user_message_count INTEGER DEFAULT 0,
|
||||
assistant_message_count INTEGER DEFAULT 0,
|
||||
total_input_tokens INTEGER DEFAULT 0,
|
||||
total_output_tokens INTEGER DEFAULT 0,
|
||||
model TEXT,
|
||||
slug TEXT,
|
||||
file_path TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
|
||||
# Messages table
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT NOT NULL,
|
||||
message_uuid TEXT UNIQUE NOT NULL,
|
||||
parent_uuid TEXT,
|
||||
role TEXT NOT NULL,
|
||||
content_text TEXT,
|
||||
timestamp DATETIME NOT NULL,
|
||||
prompt_id TEXT,
|
||||
model TEXT,
|
||||
input_tokens INTEGER,
|
||||
output_tokens INTEGER,
|
||||
has_tool_use BOOLEAN DEFAULT 0,
|
||||
has_thinking BOOLEAN DEFAULT 0,
|
||||
stop_reason TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
# Tool calls table
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS tool_calls (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
message_uuid TEXT NOT NULL,
|
||||
tool_use_id TEXT UNIQUE NOT NULL,
|
||||
tool_name TEXT NOT NULL,
|
||||
tool_input TEXT,
|
||||
tool_result TEXT,
|
||||
is_error BOOLEAN DEFAULT 0,
|
||||
timestamp DATETIME NOT NULL
|
||||
)
|
||||
""")
|
||||
|
||||
# Daily summaries table
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS daily_summaries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
date TEXT UNIQUE NOT NULL,
|
||||
project_path TEXT,
|
||||
conversation_count INTEGER,
|
||||
total_messages INTEGER,
|
||||
total_tokens INTEGER,
|
||||
summary_content TEXT NOT NULL,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
|
||||
# Checkpoints table
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS checkpoints (
|
||||
file_path TEXT PRIMARY KEY,
|
||||
last_line_count INTEGER NOT NULL,
|
||||
last_modified DATETIME NOT NULL
|
||||
)
|
||||
""")
|
||||
|
||||
# Create indexes
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_messages_timestamp ON messages(timestamp)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_tool_calls_message ON tool_calls(message_uuid)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_conversations_project ON conversations(project_path)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_conversations_started ON conversations(started_at)")
|
||||
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def upsert_conversation(session_id, data):
|
||||
"""Insert or update conversation"""
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
await db.execute("""
|
||||
INSERT INTO conversations (
|
||||
session_id, project_path, git_branch, started_at, last_updated_at,
|
||||
message_count, user_message_count, assistant_message_count,
|
||||
total_input_tokens, total_output_tokens, model, slug, file_path
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(session_id) DO UPDATE SET
|
||||
last_updated_at = excluded.last_updated_at,
|
||||
message_count = excluded.message_count,
|
||||
user_message_count = excluded.user_message_count,
|
||||
assistant_message_count = excluded.assistant_message_count,
|
||||
total_input_tokens = excluded.total_input_tokens,
|
||||
total_output_tokens = excluded.total_output_tokens,
|
||||
model = excluded.model,
|
||||
slug = excluded.slug,
|
||||
git_branch = excluded.git_branch
|
||||
""", (
|
||||
session_id, data['project_path'], data.get('git_branch'),
|
||||
data['started_at'], data['last_updated_at'],
|
||||
data['message_count'], data['user_message_count'], data['assistant_message_count'],
|
||||
data['total_input_tokens'], data['total_output_tokens'],
|
||||
data.get('model'), data.get('slug'), data['file_path']
|
||||
))
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def insert_message(message_data):
|
||||
"""Insert message (skip if exists)"""
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
try:
|
||||
await db.execute("""
|
||||
INSERT INTO messages (
|
||||
session_id, message_uuid, parent_uuid, role, content_text,
|
||||
timestamp, prompt_id, model, input_tokens, output_tokens,
|
||||
has_tool_use, has_thinking, stop_reason
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
message_data['session_id'], message_data['message_uuid'],
|
||||
message_data.get('parent_uuid'), message_data['role'],
|
||||
message_data.get('content_text'), message_data['timestamp'],
|
||||
message_data.get('prompt_id'), message_data.get('model'),
|
||||
message_data.get('input_tokens'), message_data.get('output_tokens'),
|
||||
message_data.get('has_tool_use', False), message_data.get('has_thinking', False),
|
||||
message_data.get('stop_reason')
|
||||
))
|
||||
await db.commit()
|
||||
except aiosqlite.IntegrityError:
|
||||
# Message already exists, skip
|
||||
pass
|
||||
|
||||
|
||||
async def insert_tool_call(tool_data):
|
||||
"""Insert tool call (skip if exists)"""
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
try:
|
||||
await db.execute("""
|
||||
INSERT INTO tool_calls (
|
||||
message_uuid, tool_use_id, tool_name, tool_input,
|
||||
tool_result, is_error, timestamp
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
tool_data['message_uuid'], tool_data['tool_use_id'],
|
||||
tool_data['tool_name'], tool_data.get('tool_input'),
|
||||
tool_data.get('tool_result'), tool_data.get('is_error', False),
|
||||
tool_data['timestamp']
|
||||
))
|
||||
await db.commit()
|
||||
except aiosqlite.IntegrityError:
|
||||
# Tool call already exists, skip
|
||||
pass
|
||||
|
||||
|
||||
async def get_checkpoint(file_path):
|
||||
"""Get checkpoint for a file"""
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
cursor = await db.execute(
|
||||
"SELECT last_line_count, last_modified FROM checkpoints WHERE file_path = ?",
|
||||
(file_path,)
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return {"last_line_count": row[0], "last_modified": row[1]} if row else None
|
||||
|
||||
|
||||
async def update_checkpoint(file_path, line_count, modified_time):
|
||||
"""Update checkpoint for a file"""
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
await db.execute("""
|
||||
INSERT INTO checkpoints (file_path, last_line_count, last_modified)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(file_path) DO UPDATE SET
|
||||
last_line_count = excluded.last_line_count,
|
||||
last_modified = excluded.last_modified
|
||||
""", (file_path, line_count, modified_time))
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def upsert_summary(date, data):
|
||||
"""Insert or update daily summary"""
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
await db.execute("""
|
||||
INSERT INTO daily_summaries (
|
||||
date, project_path, conversation_count, total_messages,
|
||||
total_tokens, summary_content
|
||||
) VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(date) DO UPDATE SET
|
||||
project_path = excluded.project_path,
|
||||
conversation_count = excluded.conversation_count,
|
||||
total_messages = excluded.total_messages,
|
||||
total_tokens = excluded.total_tokens,
|
||||
summary_content = excluded.summary_content,
|
||||
created_at = CURRENT_TIMESTAMP
|
||||
""", (
|
||||
date, data.get('project_path'), data['conversation_count'],
|
||||
data['total_messages'], data['total_tokens'], data['summary_content']
|
||||
))
|
||||
await db.commit()
|
||||
@@ -0,0 +1,21 @@
|
||||
services:
|
||||
claude-code-tracker:
|
||||
build: .
|
||||
container_name: claude-code-tracker
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /volume1/docker/claude-code-tracker/data:/app/data
|
||||
- /volume1/docker/claude-code-tracker/conversations:/app/conversations:ro
|
||||
environment:
|
||||
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
|
||||
- ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL:-https://www.bytecatcode.org}
|
||||
- SUMMARY_HOUR=${SUMMARY_HOUR:-23}
|
||||
- COLLECT_INTERVAL=${COLLECT_INTERVAL:-60}
|
||||
- DB_PATH=/app/data/conversations.db
|
||||
- CONVERSATIONS_PATH=/app/conversations
|
||||
- TRIGGER_PATH=/app/data/trigger
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
@@ -0,0 +1,47 @@
|
||||
import os
|
||||
import logging
|
||||
import asyncio
|
||||
|
||||
import db
|
||||
import collector
|
||||
import summarizer
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main service loop"""
|
||||
logger.info("Starting Claude Code Tracker service")
|
||||
|
||||
# Initialize database
|
||||
await db.init_db()
|
||||
logger.info("Database initialized")
|
||||
|
||||
# Create data directory if it doesn't exist
|
||||
data_dir = os.path.dirname(db.DB_PATH)
|
||||
os.makedirs(data_dir, exist_ok=True)
|
||||
|
||||
# Start collection and summarization tasks
|
||||
tasks = [
|
||||
asyncio.create_task(collector.collect_conversations()),
|
||||
asyncio.create_task(summarizer.summarize_daily())
|
||||
]
|
||||
|
||||
logger.info("Service started - collection and summarization tasks running")
|
||||
|
||||
try:
|
||||
await asyncio.gather(*tasks)
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Shutting down...")
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,238 @@
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_jsonl_file(file_path: str, start_line: int = 0) -> List[Dict[str, Any]]:
|
||||
"""Parse JSONL file from a specific line number"""
|
||||
messages = []
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
for i, line in enumerate(f):
|
||||
if i < start_line:
|
||||
continue
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
messages.append(obj)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning(f"Failed to parse line {i} in {file_path}: {e}")
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read {file_path}: {e}")
|
||||
return messages
|
||||
|
||||
|
||||
def extract_text_content(content: Any) -> str:
|
||||
"""Extract text from message content"""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
|
||||
if isinstance(content, list):
|
||||
text_parts = []
|
||||
for block in content:
|
||||
if isinstance(block, dict):
|
||||
if block.get('type') == 'text':
|
||||
text_parts.append(block.get('text', ''))
|
||||
elif block.get('type') == 'tool_result':
|
||||
# Include tool results in searchable text
|
||||
result = block.get('content', '')
|
||||
if isinstance(result, str):
|
||||
text_parts.append(result[:500]) # Limit length
|
||||
return '\n'.join(text_parts)
|
||||
|
||||
return ''
|
||||
|
||||
|
||||
def extract_tool_calls(content: Any, message_uuid: str, timestamp: str) -> List[Dict[str, Any]]:
|
||||
"""Extract tool use blocks from message content"""
|
||||
tool_calls = []
|
||||
|
||||
if not isinstance(content, list):
|
||||
return tool_calls
|
||||
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get('type') == 'tool_use':
|
||||
tool_calls.append({
|
||||
'message_uuid': message_uuid,
|
||||
'tool_use_id': block.get('id'),
|
||||
'tool_name': block.get('name'),
|
||||
'tool_input': json.dumps(block.get('input', {})),
|
||||
'tool_result': None,
|
||||
'is_error': False,
|
||||
'timestamp': timestamp
|
||||
})
|
||||
|
||||
return tool_calls
|
||||
|
||||
|
||||
def extract_tool_results(content: Any, tool_calls_map: Dict[str, Dict]) -> None:
|
||||
"""Extract tool results and match them to tool calls"""
|
||||
if not isinstance(content, list):
|
||||
return
|
||||
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get('type') == 'tool_result':
|
||||
tool_use_id = block.get('tool_use_id')
|
||||
if tool_use_id and tool_use_id in tool_calls_map:
|
||||
result_content = block.get('content', '')
|
||||
if isinstance(result_content, str):
|
||||
tool_calls_map[tool_use_id]['tool_result'] = result_content[:5000] # Limit length
|
||||
tool_calls_map[tool_use_id]['is_error'] = block.get('is_error', False)
|
||||
|
||||
|
||||
def extract_message_data(obj: Dict[str, Any], session_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Extract relevant fields from a message object"""
|
||||
msg_type = obj.get('type')
|
||||
|
||||
# Only process user and assistant messages
|
||||
if msg_type not in ['user', 'assistant']:
|
||||
return None
|
||||
|
||||
message = obj.get('message', {})
|
||||
role = message.get('role')
|
||||
|
||||
if not role or role not in ['user', 'assistant']:
|
||||
return None
|
||||
|
||||
content = message.get('content', '')
|
||||
has_tool_use = False
|
||||
has_thinking = False
|
||||
|
||||
# Check for tool use and thinking blocks
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
if isinstance(block, dict):
|
||||
if block.get('type') == 'tool_use':
|
||||
has_tool_use = True
|
||||
elif block.get('type') == 'thinking':
|
||||
has_thinking = True
|
||||
|
||||
# Extract usage metrics
|
||||
usage = message.get('usage', {})
|
||||
input_tokens = usage.get('input_tokens', 0)
|
||||
output_tokens = usage.get('output_tokens', 0)
|
||||
|
||||
return {
|
||||
'session_id': session_id,
|
||||
'message_uuid': obj.get('uuid'),
|
||||
'parent_uuid': obj.get('parentUuid'),
|
||||
'role': role,
|
||||
'content_text': extract_text_content(content),
|
||||
'timestamp': obj.get('timestamp'),
|
||||
'prompt_id': obj.get('promptId'),
|
||||
'model': message.get('model'),
|
||||
'input_tokens': input_tokens,
|
||||
'output_tokens': output_tokens,
|
||||
'has_tool_use': has_tool_use,
|
||||
'has_thinking': has_thinking,
|
||||
'stop_reason': message.get('stop_reason')
|
||||
}
|
||||
|
||||
|
||||
def calculate_conversation_stats(messages: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""Calculate aggregate statistics for a conversation"""
|
||||
stats = {
|
||||
'message_count': 0,
|
||||
'user_message_count': 0,
|
||||
'assistant_message_count': 0,
|
||||
'total_input_tokens': 0,
|
||||
'total_output_tokens': 0,
|
||||
'started_at': None,
|
||||
'last_updated_at': None,
|
||||
'model': None
|
||||
}
|
||||
|
||||
for msg in messages:
|
||||
if msg.get('role') == 'user':
|
||||
stats['user_message_count'] += 1
|
||||
elif msg.get('role') == 'assistant':
|
||||
stats['assistant_message_count'] += 1
|
||||
if not stats['model'] and msg.get('model'):
|
||||
stats['model'] = msg.get('model')
|
||||
|
||||
stats['message_count'] += 1
|
||||
stats['total_input_tokens'] += msg.get('input_tokens', 0)
|
||||
stats['total_output_tokens'] += msg.get('output_tokens', 0)
|
||||
|
||||
timestamp = msg.get('timestamp')
|
||||
if timestamp:
|
||||
if not stats['started_at'] or timestamp < stats['started_at']:
|
||||
stats['started_at'] = timestamp
|
||||
if not stats['last_updated_at'] or timestamp > stats['last_updated_at']:
|
||||
stats['last_updated_at'] = timestamp
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def parse_conversation_file(file_path: str, start_line: int = 0) -> Dict[str, Any]:
|
||||
"""Parse a conversation file and extract all data"""
|
||||
raw_messages = parse_jsonl_file(file_path, start_line)
|
||||
|
||||
if not raw_messages:
|
||||
return None
|
||||
|
||||
# Extract session metadata from first message
|
||||
session_id = None
|
||||
project_path = None
|
||||
git_branch = None
|
||||
slug = None
|
||||
|
||||
for obj in raw_messages:
|
||||
if obj.get('sessionId'):
|
||||
session_id = obj['sessionId']
|
||||
if obj.get('cwd'):
|
||||
project_path = obj['cwd']
|
||||
if obj.get('gitBranch'):
|
||||
git_branch = obj['gitBranch']
|
||||
if obj.get('slug'):
|
||||
slug = obj['slug']
|
||||
|
||||
if session_id and project_path:
|
||||
break
|
||||
|
||||
if not session_id:
|
||||
logger.warning(f"No session ID found in {file_path}")
|
||||
return None
|
||||
|
||||
# Extract messages
|
||||
messages = []
|
||||
tool_calls_map = {} # Map tool_use_id to tool call data
|
||||
|
||||
for obj in raw_messages:
|
||||
msg_data = extract_message_data(obj, session_id)
|
||||
if msg_data:
|
||||
messages.append(msg_data)
|
||||
|
||||
# Extract tool calls from assistant messages
|
||||
if msg_data['role'] == 'assistant' and msg_data['has_tool_use']:
|
||||
message_content = obj.get('message', {}).get('content', [])
|
||||
tool_calls = extract_tool_calls(message_content, msg_data['message_uuid'], msg_data['timestamp'])
|
||||
for tc in tool_calls:
|
||||
tool_calls_map[tc['tool_use_id']] = tc
|
||||
|
||||
# Extract tool results from user messages
|
||||
if obj.get('type') == 'user':
|
||||
message_content = obj.get('message', {}).get('content', [])
|
||||
extract_tool_results(message_content, tool_calls_map)
|
||||
|
||||
if not messages:
|
||||
return None
|
||||
|
||||
# Calculate stats
|
||||
stats = calculate_conversation_stats(messages)
|
||||
|
||||
return {
|
||||
'session_id': session_id,
|
||||
'project_path': project_path or 'unknown',
|
||||
'git_branch': git_branch,
|
||||
'slug': slug,
|
||||
'file_path': file_path,
|
||||
'messages': messages,
|
||||
'tool_calls': list(tool_calls_map.values()),
|
||||
**stats
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
aiosqlite==0.19.0
|
||||
anthropic==0.39.0
|
||||
httpx==0.27.0
|
||||
@@ -0,0 +1,174 @@
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
import asyncio
|
||||
import aiosqlite
|
||||
from anthropic import AsyncAnthropic
|
||||
|
||||
import db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY")
|
||||
ANTHROPIC_BASE_URL = os.environ.get("ANTHROPIC_BASE_URL", "https://www.bytecatcode.org")
|
||||
SUMMARY_HOUR = int(os.environ.get("SUMMARY_HOUR", "23"))
|
||||
TRIGGER_PATH = os.environ.get("TRIGGER_PATH", "/app/data/trigger")
|
||||
|
||||
|
||||
async def generate_summary_for_date(date_str: str):
|
||||
"""Generate summary for a specific date"""
|
||||
try:
|
||||
logger.info(f"Generating summary for {date_str}")
|
||||
|
||||
# Query conversations for this date
|
||||
async with aiosqlite.connect(db.DB_PATH) as conn:
|
||||
# Get conversations
|
||||
cursor = await conn.execute("""
|
||||
SELECT session_id, project_path, slug, message_count,
|
||||
total_input_tokens, total_output_tokens
|
||||
FROM conversations
|
||||
WHERE DATE(started_at) = ? OR DATE(last_updated_at) = ?
|
||||
ORDER BY started_at
|
||||
""", (date_str, date_str))
|
||||
conversations = await cursor.fetchall()
|
||||
|
||||
if not conversations:
|
||||
logger.info(f"No conversations found for {date_str}")
|
||||
return
|
||||
|
||||
# Get sample messages from each conversation
|
||||
conversation_summaries = []
|
||||
total_messages = 0
|
||||
total_tokens = 0
|
||||
projects = set()
|
||||
|
||||
for conv in conversations:
|
||||
session_id, project_path, slug, msg_count, input_tokens, output_tokens = conv
|
||||
projects.add(project_path)
|
||||
total_messages += msg_count
|
||||
total_tokens += (input_tokens + output_tokens)
|
||||
|
||||
# Get first few user messages to understand what was worked on
|
||||
cursor = await conn.execute("""
|
||||
SELECT content_text FROM messages
|
||||
WHERE session_id = ? AND role = 'user' AND content_text IS NOT NULL
|
||||
ORDER BY timestamp
|
||||
LIMIT 3
|
||||
""", (session_id,))
|
||||
user_messages = await cursor.fetchall()
|
||||
|
||||
# Get tool usage
|
||||
cursor = await conn.execute("""
|
||||
SELECT tool_name, COUNT(*) as count
|
||||
FROM tool_calls
|
||||
WHERE message_uuid IN (
|
||||
SELECT message_uuid FROM messages WHERE session_id = ?
|
||||
)
|
||||
GROUP BY tool_name
|
||||
ORDER BY count DESC
|
||||
LIMIT 5
|
||||
""", (session_id,))
|
||||
tools = await cursor.fetchall()
|
||||
|
||||
conversation_summaries.append({
|
||||
'slug': slug or 'Untitled',
|
||||
'project': project_path.split('/')[-1] if project_path else 'unknown',
|
||||
'messages': msg_count,
|
||||
'tokens': input_tokens + output_tokens,
|
||||
'user_messages': [msg[0][:200] for msg in user_messages if msg[0]],
|
||||
'tools': [f"{tool[0]} ({tool[1]}x)" for tool in tools]
|
||||
})
|
||||
|
||||
# Format for Claude
|
||||
formatted_convs = []
|
||||
for i, conv in enumerate(conversation_summaries, 1):
|
||||
formatted_convs.append(f"""
|
||||
**Conversation {i}: {conv['slug']}**
|
||||
- Project: {conv['project']}
|
||||
- Messages: {conv['messages']} ({conv['tokens']} tokens)
|
||||
- Initial requests: {', '.join(conv['user_messages'][:2]) if conv['user_messages'] else 'N/A'}
|
||||
- Tools used: {', '.join(conv['tools']) if conv['tools'] else 'None'}
|
||||
""")
|
||||
|
||||
prompt = f"""You are analyzing a developer's Claude Code conversations from {date_str}.
|
||||
|
||||
**Overview:**
|
||||
- Conversations: {len(conversations)}
|
||||
- Total messages: {total_messages}
|
||||
- Total tokens: {total_tokens:,}
|
||||
- Projects: {', '.join(sorted(projects))}
|
||||
|
||||
**Conversations:**
|
||||
{''.join(formatted_convs)}
|
||||
|
||||
Generate a concise daily summary covering:
|
||||
1. Main tasks and goals worked on
|
||||
2. Key decisions and solutions implemented
|
||||
3. Files and components modified (if evident from tool usage)
|
||||
4. Challenges encountered and how they were resolved
|
||||
5. Tools and patterns used
|
||||
6. Overall progress and outcomes
|
||||
|
||||
Use markdown formatting. Be specific and technical. Focus on what was accomplished. Keep it under 500 words."""
|
||||
|
||||
# Call Claude API
|
||||
client = AsyncAnthropic(api_key=ANTHROPIC_API_KEY, base_url=ANTHROPIC_BASE_URL)
|
||||
response = await client.messages.create(
|
||||
model="claude-haiku-4-5-20251001",
|
||||
max_tokens=4096,
|
||||
messages=[{"role": "user", "content": prompt}]
|
||||
)
|
||||
|
||||
summary_content = response.content[0].text
|
||||
|
||||
# Store summary
|
||||
await db.upsert_summary(date_str, {
|
||||
'project_path': ', '.join(sorted(projects)),
|
||||
'conversation_count': len(conversations),
|
||||
'total_messages': total_messages,
|
||||
'total_tokens': total_tokens,
|
||||
'summary_content': summary_content
|
||||
})
|
||||
|
||||
logger.info(f"Summary generated for {date_str}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating summary for {date_str}: {e}", exc_info=True)
|
||||
|
||||
|
||||
async def check_trigger():
|
||||
"""Check for manual trigger file"""
|
||||
if os.path.exists(TRIGGER_PATH):
|
||||
try:
|
||||
os.remove(TRIGGER_PATH)
|
||||
logger.info("Manual trigger detected")
|
||||
# Generate summary for yesterday
|
||||
yesterday = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d")
|
||||
await generate_summary_for_date(yesterday)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing trigger: {e}")
|
||||
|
||||
|
||||
async def summarize_daily():
|
||||
"""Daily summarization loop"""
|
||||
while True:
|
||||
try:
|
||||
now = datetime.now()
|
||||
|
||||
# Check for manual trigger
|
||||
await check_trigger()
|
||||
|
||||
# Check if it's time for daily summary
|
||||
if now.hour == SUMMARY_HOUR and now.minute < 5:
|
||||
yesterday = (now - timedelta(days=1)).strftime("%Y-%m-%d")
|
||||
await generate_summary_for_date(yesterday)
|
||||
|
||||
# Sleep until next hour to avoid duplicate runs
|
||||
await asyncio.sleep(3600)
|
||||
else:
|
||||
# Check every 5 minutes
|
||||
await asyncio.sleep(300)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in summarization loop: {e}", exc_info=True)
|
||||
await asyncio.sleep(300)
|
||||
Reference in New Issue
Block a user