Initial commit

compact_toon.py

@@ -0,0 +1,124 @@
+# compact_toon.py
+"""
+Ultra-compact Toon format translator for maximum token efficiency
+"""
+import re
+from typing import Dict, List, Any, Optional
+
+
+class CompactToon:
+    """Minimal Toon format translator for maximum token savings."""
+    
+    @staticmethod
+    def search_results(result_string: str, query: str = "") -> str:
+        """Convert search results to minimal Toon format."""
+        if not result_string or "No results found" in result_string:
+            return result_string
+            
+        results = result_string.split("---")
+        toon_blocks = []
+        
+        for result in results:
+            if not result.strip():
+                continue
+            toon_block = CompactToon._minimal_result(result.strip())
+            if toon_block:
+                toon_blocks.append(toon_block)
+        
+        return "\n\n".join(toon_blocks)
+    
+    @staticmethod
+    def _minimal_result(result_text: str) -> Optional[str]:
+        """Create minimal Toon block for a single result."""
+        try:
+            # Fast regex extraction
+            file_match = re.search(r"File:\s*(.+?)\n", result_text)
+            line_match = re.search(r"Line:\s*(\d+)", result_text)
+            type_match = re.search(r"Type:\s*(.+?)\n", result_text)
+            score_match = re.search(r"score:\s*([\d.]+)", result_text)
+            
+            if not file_match:
+                return None
+                
+            file_path = file_match.group(1).strip()
+            line_num = line_match.group(1) if line_match else "1"
+            
+            # Extract content efficiently
+            lines = result_text.split('\n')
+            content_lines = []
+            in_content = False
+            
+            for line in lines:
+                if in_content:
+                    content_lines.append(line)
+                elif line.strip() and not any(line.startswith(x) for x in 
+                                           ['###', 'File:', 'Line:', 'Type:', 'Language:']):
+                    in_content = True
+                    content_lines.append(line)
+            
+            content = '\n'.join(content_lines).strip()
+            
+            # Build minimal Toon format
+            toon_lines = []
+            toon_lines.append(f"file://{file_path}:{line_num}")
+            
+            if type_match:
+                toon_lines.append(f"  type: {type_match.group(1).strip()}")
+            if score_match:
+                toon_lines.append(f"  score: {score_match.group(1)}")
+            
+            toon_lines.append("  content: |")
+            
+            # Add content with minimal processing
+            for line in content.split('\n'):
+                toon_lines.append(f"    {line}")
+            
+            return "\n".join(toon_lines)
+            
+        except Exception:
+            return None
+    
+    @staticmethod
+    def references(result_string: str, symbol: str = "") -> str:
+        """Convert reference results to minimal format."""
+        if not result_string or "No references found" in result_string:
+            return result_string
+            
+        results = result_string.split("---")
+        toon_blocks = []
+        
+        for result in results:
+            if not result.strip():
+                continue
+            toon_block = CompactToon._minimal_reference(result.strip())
+            if toon_block:
+                toon_blocks.append(toon_block)
+        
+        return "\n\n".join(toon_blocks)
+    
+    @staticmethod
+    def _minimal_reference(result_text: str) -> Optional[str]:
+        """Create minimal reference block."""
+        try:
+            file_match = re.search(r"File:\s*(.+?)\n", result_text)
+            line_match = re.search(r"Line:\s*(\d+)", result_text)
+            
+            if not file_match or not line_match:
+                return None
+                
+            # Extract context efficiently
+            context_match = re.search(r"Context:\s*(.+)", result_text, re.DOTALL)
+            context = context_match.group(1).strip() if context_match else ""
+            
+            toon_lines = []
+            toon_lines.append(f"file://{file_match.group(1).strip()}:{line_match.group(1)}")
+            toon_lines.append("  type: reference")
+            toon_lines.append("  content: |")
+            
+            for line in context.split('\n'):
+                toon_lines.append(f"    {line}")
+            
+            return "\n".join(toon_lines)
+            
+        except Exception:
+            return None