From bf655edb1d82e3d703d66ada6afc7c7f180cd984 Mon Sep 17 00:00:00 2001 From: Stephen Adamson Date: Sun, 16 Nov 2025 00:29:02 +0000 Subject: [PATCH] reduce imports --- mcp_codebase.py | 160 ++++++++++++++++++++++++++++++------------------ 1 file changed, 102 insertions(+), 58 deletions(-) diff --git a/mcp_codebase.py b/mcp_codebase.py index 39bee1d..2f02f20 100644 --- a/mcp_codebase.py +++ b/mcp_codebase.py @@ -2505,7 +2505,7 @@ def parse_rust_file(filepath: Path) -> List[Dict]: @lru_cache(maxsize=1000) def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple: - """Fixed Rust parsing that properly handles imports as metadata, not separate entities.""" + """Fixed Rust parsing that aggregates imports and prioritizes code entities.""" filepath = Path(filepath_str) chunks = [] @@ -2530,19 +2530,23 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple: decls = json.loads(proc.stdout) logger.info(f" AST helper found {len(decls)} declarations") - # Collect imports first to attach to actual code entities + # Collect imports to attach to code entities, but don't create separate import chunks file_imports = [] code_decls = [] for d in decls: if d.get("type_") == "import": - file_imports.extend(d.get("parameters", [])) + # Collect import paths but don't create chunks for them + import_paths = d.get("parameters", []) + file_imports.extend(import_paths) + logger.debug(f" Found import: {import_paths}") else: + # Only process actual code entities code_decls.append(d) - logger.info(f" Found {len(file_imports)} imports and {len(code_decls)} code declarations") + logger.info(f" Found {len(file_imports)} imports to attach to {len(code_decls)} code entities") - # Process actual code entities (functions, structs, etc.) + # Process ACTUAL CODE entities (functions, structs, etc.) for i, d in enumerate(code_decls): start_line = max(1, d.get("start_line", 1)) end_line = max(start_line, d.get("end_line", start_line + 1)) @@ -2555,26 +2559,34 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple: chunk_code = "".join(lines[start_line-1:end_line]) + # Skip if we don't have meaningful content + if not chunk_code.strip() or len(chunk_code.strip()) < 10: + continue + metadata = { "file": str(filepath), "name": d.get("name", "") or filepath.stem, "type": d.get("type_", "") or "unknown", "line": start_line, "language": "rust", - "imports": file_imports, # Attach imports to ALL code entities in this file + "imports": file_imports, # Attach ALL file imports to each code entity } # Enhanced metadata extraction for code entities enhanced_fields = [ "visibility", "is_async", "is_unsafe", "generics", "traits", "fields", "methods", "return_type", "parameters", "docstring", - "calls" # calls from the specific entity + "calls" ] for field in enhanced_fields: if field in d and d[field] is not None: metadata[field] = d[field] + # Skip if it's just a module declaration without real content + if metadata["type"] == "module" and not metadata.get("docstring") and len(chunk_code.strip()) < 20: + continue + logger.info(f" Code {i+1}: {metadata.get('type', 'unknown')} {metadata.get('name', 'unnamed')}") # Build rich chunk text for CODE entities @@ -2584,73 +2596,79 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple: "metadata": metadata }) - # If we have no code declarations but have imports, create a file-level chunk - if not chunks and file_imports: - logger.info(" Creating file-level chunk for imports-only file") - chunks.append(create_file_level_chunk(filepath, file_imports)) + # If we have no code declarations but have meaningful content, create file-level chunk + if not chunks and len(code.strip()) > 100: + logger.info(" Creating file-level chunk for content-rich file") + chunks.append(create_file_level_chunk(filepath, file_imports, code)) except Exception as e: logger.warning(f"Rust AST helper failed for {filepath}: {e}") return create_fallback_chunk(filepath) if chunks: - logger.info(f"✅ Successfully parsed {len(chunks)} chunks from {filepath}") + logger.info(f"✅ Successfully parsed {len(chunks)} code chunks from {filepath}") return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) - logger.warning(f"❌ No chunks created from Rust AST for {filepath}. Emitting fallback.") + logger.warning(f"❌ No meaningful code chunks from Rust AST for {filepath}. Emitting fallback.") return create_fallback_chunk(filepath) def build_rust_code_chunk_text(metadata: dict, code: str) -> str: - """Build comprehensive Rust code chunk text.""" + """Build comprehensive Rust code chunk text - focused on ACTUAL CODE.""" parts = [ f"File: {metadata.get('file', '')}", f"Type: {metadata.get('type', 'unknown')}", f"Name: {metadata.get('name', '')}", ] - # Add enhanced metadata for CODE entities + # Only include the most valuable metadata if metadata.get("visibility"): parts.append(f"Visibility: {metadata['visibility']}") - if metadata.get("is_async"): - parts.append("Async: yes") - if metadata.get("is_unsafe"): - parts.append("Unsafe: yes") if metadata.get("generics"): parts.append(f"Generics: {', '.join(metadata['generics'])}") if metadata.get("traits"): parts.append(f"Implements: {', '.join(metadata['traits'])}") if metadata.get("fields"): parts.append(f"Fields: {', '.join(metadata['fields'])}") - if metadata.get("imports"): - parts.append(f"Imports: {', '.join(metadata['imports'])}") - if metadata.get("calls"): - parts.append(f"Calls: {', '.join(metadata['calls'])}") if metadata.get("docstring"): - parts.append(f"Doc: {metadata['docstring']}") + # Only include substantial docstrings + doc = metadata['docstring'] + if len(doc) > 20 and len(doc) < 300: + parts.append(f"Doc: {doc}") + + # Include imports and calls only if they exist and are meaningful + meaningful_imports = [imp for imp in metadata.get("imports", []) + if len(imp) > 3 and not imp.startswith(('c:', 'b:', 'p:', 'r:', 't:', 'v:'))] + if meaningful_imports: + parts.append(f"Imports: {', '.join(meaningful_imports[:5])}") # Limit to top 5 + + meaningful_calls = [call for call in metadata.get("calls", []) + if len(call) > 2 and not call in ['if', 'for', 'let', 'mut', 'pub']] + if meaningful_calls: + parts.append(f"Calls: {', '.join(meaningful_calls[:5])}") # Limit to top 5 parts.append(f"Code:\n{code}") return "\n".join(parts) -def create_file_level_chunk(filepath: Path, imports: list) -> dict: - """Create a chunk for files that only have imports.""" - try: - full_code = filepath.read_text(encoding="utf-8") - except Exception: - full_code = "" - +def create_file_level_chunk(filepath: Path, imports: list, code: str) -> dict: + """Create a file-level chunk only for files with substantial content.""" metadata = { "file": str(filepath), "name": filepath.stem, - "type": "module", # Treat import-only files as modules + "type": "file", "line": 1, "language": "rust", "imports": imports, } - chunk_text = f"File: {filepath}\nType: module\nName: {filepath.stem}\nImports: {', '.join(imports)}\nCode:\n{full_code}" + # Only include meaningful imports + meaningful_imports = [imp for imp in imports if len(imp) > 3] + + chunk_text = f"File: {filepath}\nType: file\nName: {filepath.stem}" + if meaningful_imports: + chunk_text += f"\nImports: {', '.join(meaningful_imports[:3])}" + chunk_text += f"\nCode:\n{code}" return {"text": chunk_text, "metadata": metadata} - def create_fallback_chunk(filepath: Path) -> tuple: """Create a minimal fallback chunk with proper metadata.""" try: @@ -3573,44 +3591,70 @@ def build_indexes(): # ------------------------------------------------- def _add_import_edges_enhanced(graph, file_node_id, importing_node_id, imports, lang): - """Enhanced import edge creation that resolves Rust paths to actual files.""" + """Enhanced import edge creation that avoids symbolic node spam.""" new_nodes = {} for imp_path in imports: if not imp_path or not isinstance(imp_path, str): continue - # Skip standard library and external crates for now - if any(imp_path.startswith(prefix) for prefix in ['std::', 'core::', 'alloc::', 'bevy::', 'serde::']): + # Skip single-letter and obviously symbolic imports + if len(imp_path.strip()) <= 1 or imp_path in ['c', 'b', 'p', 'r', 't', 'v', 'd', 'm', 'q', 's', 'n', 'f', 'a']: continue - # Try to resolve relative paths within the codebase + # Skip common standard library and external crates (they create too much noise) + external_prefixes = ['std::', 'core::', 'alloc::', 'bevy::', 'serde::', 'rayon::', 'rand::'] + if any(imp_path.startswith(prefix) for prefix in external_prefixes): + # Only create ONE symbolic node for the entire crate, not individual items + crate_name = imp_path.split('::')[0] + if crate_name: + symbolic_id = f"rust::external::{crate_name}" + if symbolic_id not in graph.graph.nodes: + graph.add_node( + symbolic_id, + type="external_crate", + name=crate_name, + file="", + lang="rust", + is_external=True + ) + new_nodes[symbolic_id] = { + "type": "external_crate", + "name": crate_name, + "file": "", + "lang": "rust" + } + if not graph.graph.has_edge(importing_node_id, symbolic_id): + graph.add_edge(importing_node_id, symbolic_id, "imports") + continue + + # For internal imports, try to resolve to actual files resolved_path = resolve_rust_import(imp_path, graph.root_path) if resolved_path: - # Create edge to the actual file target_file_node_id = f"file::{resolved_path}" if not graph.graph.has_edge(importing_node_id, target_file_node_id): graph.add_edge(importing_node_id, target_file_node_id, "imports") else: - # Create symbolic node for external/unresolved imports - symbolic_id = f"rust::external::{imp_path}" - if symbolic_id not in graph.graph.nodes: - graph.add_node( - symbolic_id, - type="external", - name=imp_path, - file="", - lang="rust", - is_external=True - ) - new_nodes[symbolic_id] = { - "type": "external", - "name": imp_path, - "file": "", - "lang": "rust" - } - if not graph.graph.has_edge(importing_node_id, symbolic_id): - graph.add_edge(importing_node_id, symbolic_id, "imports") + # Only create symbolic nodes for substantial internal paths + if '::' in imp_path and len(imp_path) > 5: + symbolic_id = f"rust::symbol::{imp_path}" + if symbolic_id not in graph.graph.nodes: + graph.add_node( + symbolic_id, + type="symbol", + name=imp_path, + file="", + lang="rust", + is_symbolic=True + ) + new_nodes[symbolic_id] = { + "type": "symbol", + "name": imp_path, + "file": "", + "lang": "rust" + } + if not graph.graph.has_edge(importing_node_id, symbolic_id): + graph.add_edge(importing_node_id, symbolic_id, "imports") return new_nodes