fix: add batch processing for large conversation files

2026-04-12 11:33:24 +08:00
parent 5579deb433
commit c27cc505e1
2 changed files with 15 additions and 9 deletions
@@ -52,9 +52,13 @@ async def process_file(file_path: str):
            # No new lines
            return 0

+        # Process in batches for large files (max 200 lines at a time)
+        batch_size = 200
+        end_line = min(start_line + batch_size, total_lines)
+
        # Parse new content
-        logger.info(f"Processing {file_path} from line {start_line} to {total_lines}")
-        conversation_data = parser.parse_conversation_file(file_path, start_line)
+        logger.info(f"Processing {file_path} from line {start_line} to {end_line} (total: {total_lines})")
+        conversation_data = parser.parse_conversation_file(file_path, start_line, end_line)

        if not conversation_data:
            logger.warning(f"No data extracted from {file_path}")
@@ -87,10 +91,10 @@ async def process_file(file_path: str):
        for tool_call in conversation_data['tool_calls']:
            await db.insert_tool_call(tool_call)

-        # Update checkpoint
-        await db.update_checkpoint(file_path, total_lines, file_mtime)
+        # Update checkpoint (use end_line instead of total_lines for batch processing)
+        await db.update_checkpoint(file_path, end_line, file_mtime)

-        logger.info(f"Processed {len(conversation_data['messages'])} messages from {file_path}")
+        logger.info(f"Processed {len(conversation_data['messages'])} messages from {file_path} (batch {start_line}-{end_line}/{total_lines})")
        return len(conversation_data['messages'])

    except Exception as e: