fix: add batch processing for large conversation files
This commit is contained in:
@@ -6,14 +6,16 @@ from typing import Dict, List, Any, Optional
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_jsonl_file(file_path: str, start_line: int = 0) -> List[Dict[str, Any]]:
|
||||
"""Parse JSONL file from a specific line number"""
|
||||
def parse_jsonl_file(file_path: str, start_line: int = 0, end_line: int = None) -> List[Dict[str, Any]]:
|
||||
"""Parse JSONL file from a specific line number to end_line (or EOF if None)"""
|
||||
messages = []
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
for i, line in enumerate(f):
|
||||
if i < start_line:
|
||||
continue
|
||||
if end_line is not None and i >= end_line:
|
||||
break
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
@@ -169,9 +171,9 @@ def calculate_conversation_stats(messages: List[Dict[str, Any]]) -> Dict[str, An
|
||||
return stats
|
||||
|
||||
|
||||
def parse_conversation_file(file_path: str, start_line: int = 0) -> Dict[str, Any]:
|
||||
def parse_conversation_file(file_path: str, start_line: int = 0, end_line: int = None) -> Dict[str, Any]:
|
||||
"""Parse a conversation file and extract all data"""
|
||||
raw_messages = parse_jsonl_file(file_path, start_line)
|
||||
raw_messages = parse_jsonl_file(file_path, start_line, end_line)
|
||||
|
||||
if not raw_messages:
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user