125 lines
4.4 KiB
Python
125 lines
4.4 KiB
Python
# compact_toon.py
|
|
"""
|
|
Ultra-compact Toon format translator for maximum token efficiency
|
|
"""
|
|
import re
|
|
from typing import Dict, List, Any, Optional
|
|
|
|
|
|
class CompactToon:
|
|
"""Minimal Toon format translator for maximum token savings."""
|
|
|
|
@staticmethod
|
|
def search_results(result_string: str, query: str = "") -> str:
|
|
"""Convert search results to minimal Toon format."""
|
|
if not result_string or "No results found" in result_string:
|
|
return result_string
|
|
|
|
results = result_string.split("---")
|
|
toon_blocks = []
|
|
|
|
for result in results:
|
|
if not result.strip():
|
|
continue
|
|
toon_block = CompactToon._minimal_result(result.strip())
|
|
if toon_block:
|
|
toon_blocks.append(toon_block)
|
|
|
|
return "\n\n".join(toon_blocks)
|
|
|
|
@staticmethod
|
|
def _minimal_result(result_text: str) -> Optional[str]:
|
|
"""Create minimal Toon block for a single result."""
|
|
try:
|
|
# Fast regex extraction
|
|
file_match = re.search(r"File:\s*(.+?)\n", result_text)
|
|
line_match = re.search(r"Line:\s*(\d+)", result_text)
|
|
type_match = re.search(r"Type:\s*(.+?)\n", result_text)
|
|
score_match = re.search(r"score:\s*([\d.]+)", result_text)
|
|
|
|
if not file_match:
|
|
return None
|
|
|
|
file_path = file_match.group(1).strip()
|
|
line_num = line_match.group(1) if line_match else "1"
|
|
|
|
# Extract content efficiently
|
|
lines = result_text.split('\n')
|
|
content_lines = []
|
|
in_content = False
|
|
|
|
for line in lines:
|
|
if in_content:
|
|
content_lines.append(line)
|
|
elif line.strip() and not any(line.startswith(x) for x in
|
|
['###', 'File:', 'Line:', 'Type:', 'Language:']):
|
|
in_content = True
|
|
content_lines.append(line)
|
|
|
|
content = '\n'.join(content_lines).strip()
|
|
|
|
# Build minimal Toon format
|
|
toon_lines = []
|
|
toon_lines.append(f"file://{file_path}:{line_num}")
|
|
|
|
if type_match:
|
|
toon_lines.append(f" type: {type_match.group(1).strip()}")
|
|
if score_match:
|
|
toon_lines.append(f" score: {score_match.group(1)}")
|
|
|
|
toon_lines.append(" content: |")
|
|
|
|
# Add content with minimal processing
|
|
for line in content.split('\n'):
|
|
toon_lines.append(f" {line}")
|
|
|
|
return "\n".join(toon_lines)
|
|
|
|
except Exception:
|
|
return None
|
|
|
|
@staticmethod
|
|
def references(result_string: str, symbol: str = "") -> str:
|
|
"""Convert reference results to minimal format."""
|
|
if not result_string or "No references found" in result_string:
|
|
return result_string
|
|
|
|
results = result_string.split("---")
|
|
toon_blocks = []
|
|
|
|
for result in results:
|
|
if not result.strip():
|
|
continue
|
|
toon_block = CompactToon._minimal_reference(result.strip())
|
|
if toon_block:
|
|
toon_blocks.append(toon_block)
|
|
|
|
return "\n\n".join(toon_blocks)
|
|
|
|
@staticmethod
|
|
def _minimal_reference(result_text: str) -> Optional[str]:
|
|
"""Create minimal reference block."""
|
|
try:
|
|
file_match = re.search(r"File:\s*(.+?)\n", result_text)
|
|
line_match = re.search(r"Line:\s*(\d+)", result_text)
|
|
|
|
if not file_match or not line_match:
|
|
return None
|
|
|
|
# Extract context efficiently
|
|
context_match = re.search(r"Context:\s*(.+)", result_text, re.DOTALL)
|
|
context = context_match.group(1).strip() if context_match else ""
|
|
|
|
toon_lines = []
|
|
toon_lines.append(f"file://{file_match.group(1).strip()}:{line_match.group(1)}")
|
|
toon_lines.append(" type: reference")
|
|
toon_lines.append(" content: |")
|
|
|
|
for line in context.split('\n'):
|
|
toon_lines.append(f" {line}")
|
|
|
|
return "\n".join(toon_lines)
|
|
|
|
except Exception:
|
|
return None
|