# enhanced_toon.py """ Official TOON (Token-Oriented Object Notation) formatter following the spec from github.com/toon-format/toon Provides 30-60% token savings compared to JSON for LLM communication. """ import re from typing import List, Dict, Any, Optional class EnhancedToon: """ TOON formatter implementing the official specification from toon-format/toon Optimized for uniform arrays of objects with significant token reduction. """ @staticmethod def search_results(search_output: str, query: str) -> str: """ Convert search results to proper TOON format following official spec. Args: search_output: Raw search results from search_codebase query: Original search query Returns: Proper TOON formatted results """ if not search_output or "No results" in search_output: return f"# Search: {query}\nNo results found.\n" # If it's already an error message, return as is if "Error reading file" in search_output: return search_output try: # Parse individual results results = [] current_result = {} lines = search_output.split('\n') i = 0 while i < len(lines): line = lines[i].strip() if line.startswith('### Result'): # Save previous result if exists if current_result and current_result.get('file'): results.append(current_result) # Start new result current_result = { 'file': '', 'line': '', 'type': '', 'content': [] } # Extract score from header (remove relevance text, keep only score) score_match = re.search(r'score:\s*([\d.]+)', line) if score_match: current_result['score'] = float(score_match.group(1)) else: current_result['score'] = 0.0 i += 1 elif line.startswith('File:') and current_result: # Only take the first File: line, ignore duplicates if not current_result['file']: current_result['file'] = line.replace('File:', '').strip() i += 1 elif line.startswith('Line:') and current_result: # Only take the first Line: line, ignore duplicates if not current_result['line']: current_result['line'] = line.replace('Line:', '').strip() i += 1 elif line.startswith('Type:') and current_result: # Only take the first Type: line, ignore duplicates if not current_result['type']: current_result['type'] = line.replace('Type:', '').strip() i += 1 elif line == '---': # End of result i += 1 elif line and not line.startswith('###') and current_result: # Skip all metadata lines (File:, Type:, Name:, Language:, Doc:, SQL:, Code:) if not any(line.startswith(prefix) for prefix in ['File:', 'Line:', 'Type:', 'Name:', 'Language:', 'Doc:', 'SQL:', 'Code:']): current_result['content'].append(line) i += 1 else: i += 1 # Add the last result if valid if current_result and current_result.get('file'): results.append(current_result) # Convert to proper TOON format return EnhancedToon._format_search_toon(results, query) except Exception as e: # If parsing fails, return original output but clean it up return EnhancedToon._clean_raw_output(search_output, query) @staticmethod def _clean_raw_output(raw_output: str, query: str) -> str: """ Fallback: Clean up raw output by removing duplicate metadata lines. """ lines = raw_output.split('\n') cleaned_lines = [] skip_next_metadata = False for line in lines: stripped = line.strip() # Skip duplicate metadata lines that appear after the first occurrence if any(stripped.startswith(prefix) for prefix in ['File:', 'Type:', 'Name:', 'Language:', 'Doc:']): if skip_next_metadata: continue skip_next_metadata = True elif stripped.startswith('### Result') or stripped == '---': skip_next_metadata = False cleaned_lines.append(line) else: cleaned_lines.append(line) # Also remove relevance text, keep only score final_lines = [] for line in cleaned_lines: # Remove "relevance: HIGH/MEDIUM/LOW" text, keep only score line = re.sub(r'relevance:\s*\w+,\s*', '', line) final_lines.append(line) return "\n".join(final_lines) @staticmethod def _format_search_toon(results: List[Dict], query: str) -> str: """ Format search results following official TOON specification. """ if not results: return f"# Search: {query}\nNo results found.\n" lines = [f"# Search: {query}", f"results[{len(results)}]{{file,line,score,content}}:"] for i, result in enumerate(results, 1): # Prepare content - join and clean content = ' '.join(result.get('content', [])).strip() content = re.sub(r'\s+', ' ', content) # Normalize whitespace # Remove any remaining metadata patterns content = re.sub(r'^File:.*$', '', content, flags=re.MULTILINE) content = re.sub(r'^Type:.*$', '', content, flags=re.MULTILINE) content = re.sub(r'^Name:.*$', '', content, flags=re.MULTILINE) content = re.sub(r'^Language:.*$', '', content, flags=re.MULTILINE) content = re.sub(r'^Doc:.*$', '', content, flags=re.MULTILINE) content = re.sub(r'^SQL:.*$', '', content, flags=re.MULTILINE) content = re.sub(r'^Code:.*$', '', content, flags=re.MULTILINE) content = content.strip() # Build TOON row - simple format without relevance text row = [ result.get('file', ''), result.get('line', ''), f"{result.get('score', 0.0):.2f}", content[:300] # Reasonable limit ] # Escape fields if needed and join with comma escaped_row = [EnhancedToon._escape_toon_field(str(field)) for field in row] lines.append(" " + ",".join(escaped_row)) return "\n".join(lines) @staticmethod def _escape_toon_field(field: str) -> str: """ Escape TOON field according to specification. Only quote fields that contain commas, quotes, or newlines. """ if any(char in field for char in [',', '"', '\n', '\r']): # Escape quotes and wrap in quotes escaped = field.replace('"', '\\"') return f'"{escaped}"' return field @staticmethod def reference_results(reference_output: str, symbol: str) -> str: """ Convert reference results to clean format. """ if "Error reading file" in reference_output: return reference_output # Clean up reference output similarly return EnhancedToon._clean_raw_output(reference_output, f"References: {symbol}") @staticmethod def file_content_results(file_output: str, path: str) -> str: """ Convert file content results, preserving the full content without truncation. """ # Remove any truncation messages and return full content if "truncated to 100 lines" in file_output: # This indicates the file was truncated, we want to avoid that # For now, just return the original output but remove the truncation notice lines = file_output.split('\n') cleaned_lines = [] for line in lines: if "truncated to" not in line and "Context:" not in line: cleaned_lines.append(line) return "\n".join(cleaned_lines) return file_output