reduce imports

mcp_codebase.py

@@ -2505,7 +2505,7 @@ def parse_rust_file(filepath: Path) -> List[Dict]:
 
 @lru_cache(maxsize=1000)
 def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
-    """Fixed Rust parsing that properly handles imports as metadata, not separate entities."""
+    """Fixed Rust parsing that aggregates imports and prioritizes code entities."""
     filepath = Path(filepath_str)
     chunks = []
 
@@ -2530,19 +2530,23 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
         decls = json.loads(proc.stdout)
         logger.info(f"  AST helper found {len(decls)} declarations")
 
-        # Collect imports first to attach to actual code entities
+        # Collect imports to attach to code entities, but don't create separate import chunks
         file_imports = []
         code_decls = []
 
         for d in decls:
             if d.get("type_") == "import":
-                file_imports.extend(d.get("parameters", []))
+                # Collect import paths but don't create chunks for them
+                import_paths = d.get("parameters", [])
+                file_imports.extend(import_paths)
+                logger.debug(f"  Found import: {import_paths}")
             else:
+                # Only process actual code entities
                 code_decls.append(d)
 
-        logger.info(f"  Found {len(file_imports)} imports and {len(code_decls)} code declarations")
+        logger.info(f"  Found {len(file_imports)} imports to attach to {len(code_decls)} code entities")
 
-        # Process actual code entities (functions, structs, etc.)
+        # Process ACTUAL CODE entities (functions, structs, etc.)
         for i, d in enumerate(code_decls):
             start_line = max(1, d.get("start_line", 1))
             end_line = max(start_line, d.get("end_line", start_line + 1))
@@ -2555,26 +2559,34 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
 
             chunk_code = "".join(lines[start_line-1:end_line])
 
+            # Skip if we don't have meaningful content
+            if not chunk_code.strip() or len(chunk_code.strip()) < 10:
+                continue
+
             metadata = {
                 "file": str(filepath),
                 "name": d.get("name", "") or filepath.stem,
                 "type": d.get("type_", "") or "unknown",
                 "line": start_line,
                 "language": "rust",
-                "imports": file_imports,  # Attach imports to ALL code entities in this file
+                "imports": file_imports,  # Attach ALL file imports to each code entity
             }
 
             # Enhanced metadata extraction for code entities
             enhanced_fields = [
                 "visibility", "is_async", "is_unsafe", "generics", "traits",
                 "fields", "methods", "return_type", "parameters", "docstring",
-                "calls"  # calls from the specific entity
+                "calls"
             ]
 
             for field in enhanced_fields:
                 if field in d and d[field] is not None:
                     metadata[field] = d[field]
 
+            # Skip if it's just a module declaration without real content
+            if metadata["type"] == "module" and not metadata.get("docstring") and len(chunk_code.strip()) < 20:
+                continue
+
             logger.info(f"    Code {i+1}: {metadata.get('type', 'unknown')} {metadata.get('name', 'unnamed')}")
 
             # Build rich chunk text for CODE entities
@@ -2584,73 +2596,79 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
                 "metadata": metadata
             })
 
-        # If we have no code declarations but have imports, create a file-level chunk
+        # If we have no code declarations but have meaningful content, create file-level chunk
-        if not chunks and file_imports:
+        if not chunks and len(code.strip()) > 100:
-            logger.info("  Creating file-level chunk for imports-only file")
+            logger.info("  Creating file-level chunk for content-rich file")
-            chunks.append(create_file_level_chunk(filepath, file_imports))
+            chunks.append(create_file_level_chunk(filepath, file_imports, code))
 
     except Exception as e:
         logger.warning(f"Rust AST helper failed for {filepath}: {e}")
         return create_fallback_chunk(filepath)
 
     if chunks:
-        logger.info(f"✅ Successfully parsed {len(chunks)} chunks from {filepath}")
+        logger.info(f"✅ Successfully parsed {len(chunks)} code chunks from {filepath}")
         return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks)
 
-    logger.warning(f"❌ No chunks created from Rust AST for {filepath}. Emitting fallback.")
+    logger.warning(f"❌ No meaningful code chunks from Rust AST for {filepath}. Emitting fallback.")
     return create_fallback_chunk(filepath)
 
 def build_rust_code_chunk_text(metadata: dict, code: str) -> str:
-    """Build comprehensive Rust code chunk text."""
+    """Build comprehensive Rust code chunk text - focused on ACTUAL CODE."""
     parts = [
         f"File: {metadata.get('file', '')}",
         f"Type: {metadata.get('type', 'unknown')}",
         f"Name: {metadata.get('name', '')}",
     ]
 
-    # Add enhanced metadata for CODE entities
+    # Only include the most valuable metadata
     if metadata.get("visibility"):
         parts.append(f"Visibility: {metadata['visibility']}")
-    if metadata.get("is_async"):
-        parts.append("Async: yes")
-    if metadata.get("is_unsafe"):
-        parts.append("Unsafe: yes")
     if metadata.get("generics"):
         parts.append(f"Generics: {', '.join(metadata['generics'])}")
     if metadata.get("traits"):
         parts.append(f"Implements: {', '.join(metadata['traits'])}")
     if metadata.get("fields"):
         parts.append(f"Fields: {', '.join(metadata['fields'])}")
-    if metadata.get("imports"):
-        parts.append(f"Imports: {', '.join(metadata['imports'])}")
-    if metadata.get("calls"):
-        parts.append(f"Calls: {', '.join(metadata['calls'])}")
     if metadata.get("docstring"):
-        parts.append(f"Doc: {metadata['docstring']}")
+        # Only include substantial docstrings
+        doc = metadata['docstring']
+        if len(doc) > 20 and len(doc) < 300:
+            parts.append(f"Doc: {doc}")
+
+    # Include imports and calls only if they exist and are meaningful
+    meaningful_imports = [imp for imp in metadata.get("imports", [])
+                         if len(imp) > 3 and not imp.startswith(('c:', 'b:', 'p:', 'r:', 't:', 'v:'))]
+    if meaningful_imports:
+        parts.append(f"Imports: {', '.join(meaningful_imports[:5])}")  # Limit to top 5
+
+    meaningful_calls = [call for call in metadata.get("calls", [])
+                       if len(call) > 2 and not call in ['if', 'for', 'let', 'mut', 'pub']]
+    if meaningful_calls:
+        parts.append(f"Calls: {', '.join(meaningful_calls[:5])}")  # Limit to top 5
 
     parts.append(f"Code:\n{code}")
     return "\n".join(parts)
 
-def create_file_level_chunk(filepath: Path, imports: list) -> dict:
+def create_file_level_chunk(filepath: Path, imports: list, code: str) -> dict:
-    """Create a chunk for files that only have imports."""
+    """Create a file-level chunk only for files with substantial content."""
-    try:
-        full_code = filepath.read_text(encoding="utf-8")
-    except Exception:
-        full_code = ""
-
     metadata = {
         "file": str(filepath),
         "name": filepath.stem,
-        "type": "module",  # Treat import-only files as modules
+        "type": "file",
         "line": 1,
         "language": "rust",
         "imports": imports,
     }
 
-    chunk_text = f"File: {filepath}\nType: module\nName: {filepath.stem}\nImports: {', '.join(imports)}\nCode:\n{full_code}"
+    # Only include meaningful imports
+    meaningful_imports = [imp for imp in imports if len(imp) > 3]
+
+    chunk_text = f"File: {filepath}\nType: file\nName: {filepath.stem}"
+    if meaningful_imports:
+        chunk_text += f"\nImports: {', '.join(meaningful_imports[:3])}"
+    chunk_text += f"\nCode:\n{code}"
 
     return {"text": chunk_text, "metadata": metadata}
-
 def create_fallback_chunk(filepath: Path) -> tuple:
     """Create a minimal fallback chunk with proper metadata."""
     try:
@@ -3573,44 +3591,70 @@ def build_indexes():
 # -------------------------------------------------
 
 def _add_import_edges_enhanced(graph, file_node_id, importing_node_id, imports, lang):
-    """Enhanced import edge creation that resolves Rust paths to actual files."""
+    """Enhanced import edge creation that avoids symbolic node spam."""
     new_nodes = {}
 
     for imp_path in imports:
         if not imp_path or not isinstance(imp_path, str):
             continue
 
-        # Skip standard library and external crates for now
+        # Skip single-letter and obviously symbolic imports
-        if any(imp_path.startswith(prefix) for prefix in ['std::', 'core::', 'alloc::', 'bevy::', 'serde::']):
+        if len(imp_path.strip()) <= 1 or imp_path in ['c', 'b', 'p', 'r', 't', 'v', 'd', 'm', 'q', 's', 'n', 'f', 'a']:
             continue
 
-        # Try to resolve relative paths within the codebase
+        # Skip common standard library and external crates (they create too much noise)
+        external_prefixes = ['std::', 'core::', 'alloc::', 'bevy::', 'serde::', 'rayon::', 'rand::']
+        if any(imp_path.startswith(prefix) for prefix in external_prefixes):
+            # Only create ONE symbolic node for the entire crate, not individual items
+            crate_name = imp_path.split('::')[0]
+            if crate_name:
+                symbolic_id = f"rust::external::{crate_name}"
+                if symbolic_id not in graph.graph.nodes:
+                    graph.add_node(
+                        symbolic_id,
+                        type="external_crate",
+                        name=crate_name,
+                        file="",
+                        lang="rust",
+                        is_external=True
+                    )
+                    new_nodes[symbolic_id] = {
+                        "type": "external_crate",
+                        "name": crate_name,
+                        "file": "",
+                        "lang": "rust"
+                    }
+                if not graph.graph.has_edge(importing_node_id, symbolic_id):
+                    graph.add_edge(importing_node_id, symbolic_id, "imports")
+            continue
+
+        # For internal imports, try to resolve to actual files
         resolved_path = resolve_rust_import(imp_path, graph.root_path)
         if resolved_path:
-            # Create edge to the actual file
             target_file_node_id = f"file::{resolved_path}"
             if not graph.graph.has_edge(importing_node_id, target_file_node_id):
                 graph.add_edge(importing_node_id, target_file_node_id, "imports")
         else:
-            # Create symbolic node for external/unresolved imports
+            # Only create symbolic nodes for substantial internal paths
-            symbolic_id = f"rust::external::{imp_path}"
+            if '::' in imp_path and len(imp_path) > 5:
-            if symbolic_id not in graph.graph.nodes:
+                symbolic_id = f"rust::symbol::{imp_path}"
-                graph.add_node(
+                if symbolic_id not in graph.graph.nodes:
-                    symbolic_id,
+                    graph.add_node(
-                    type="external",
+                        symbolic_id,
-                    name=imp_path,
+                        type="symbol",
-                    file="",
+                        name=imp_path,
-                    lang="rust",
+                        file="",
-                    is_external=True
+                        lang="rust",
-                )
+                        is_symbolic=True
-                new_nodes[symbolic_id] = {
+                    )
-                    "type": "external",
+                    new_nodes[symbolic_id] = {
-                    "name": imp_path,
+                        "type": "symbol",
-                    "file": "",
+                        "name": imp_path,
-                    "lang": "rust"
+                        "file": "",
-                }
+                        "lang": "rust"
-            if not graph.graph.has_edge(importing_node_id, symbolic_id):
+                    }
-                graph.add_edge(importing_node_id, symbolic_id, "imports")
+                if not graph.graph.has_edge(importing_node_id, symbolic_id):
+                    graph.add_edge(importing_node_id, symbolic_id, "imports")
 
     return new_nodes