fix: add batch processing for large conversation files
Run Tests / Backend Tests (pull_request) Failing after 5m48s
Run Tests / Frontend Tests (pull_request) Failing after 2m1s
Run Tests / Test Summary (pull_request) Failing after 15s

This commit is contained in:
Gan, Jimmy
2026-04-12 11:33:24 +08:00
parent 5579deb433
commit c27cc505e1
2 changed files with 15 additions and 9 deletions
+6 -4
View File
@@ -6,14 +6,16 @@ from typing import Dict, List, Any, Optional
logger = logging.getLogger(__name__)
def parse_jsonl_file(file_path: str, start_line: int = 0) -> List[Dict[str, Any]]:
"""Parse JSONL file from a specific line number"""
def parse_jsonl_file(file_path: str, start_line: int = 0, end_line: int = None) -> List[Dict[str, Any]]:
"""Parse JSONL file from a specific line number to end_line (or EOF if None)"""
messages = []
try:
with open(file_path, 'r', encoding='utf-8') as f:
for i, line in enumerate(f):
if i < start_line:
continue
if end_line is not None and i >= end_line:
break
if not line.strip():
continue
try:
@@ -169,9 +171,9 @@ def calculate_conversation_stats(messages: List[Dict[str, Any]]) -> Dict[str, An
return stats
def parse_conversation_file(file_path: str, start_line: int = 0) -> Dict[str, Any]:
def parse_conversation_file(file_path: str, start_line: int = 0, end_line: int = None) -> Dict[str, Any]:
"""Parse a conversation file and extract all data"""
raw_messages = parse_jsonl_file(file_path, start_line)
raw_messages = parse_jsonl_file(file_path, start_line, end_line)
if not raw_messages:
return None