re-do rust

mcp_codebase.py

@@ -290,6 +290,12 @@ def apply_contextual_weights_to_embeddings(embeddings_list: List[List[float]],
     """
     weighted_embeddings = []
 
+    if len(embeddings_list) != len(metadata_list):
+        raise ValueError(
+            f"Embedding/metadata mismatch: {len(embeddings_list)} embeddings vs {len(metadata_list)} metadata. "
+            "This will break downstream systems (Chroma)."
+        )
+
     for embedding, metadata in zip(embeddings_list, metadata_list):
         weight = calculate_contextual_weight(metadata)
 
@@ -1543,32 +1549,33 @@ def parse_rust_file(filepath: Path) -> List[Dict]:
 
 @lru_cache(maxsize=1000)
 def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
-    """Cached Rust parsing with enhanced metadata for graph relationships."""
+    """Cached Rust parsing with enhanced metadata for graph relationships.
+    ALWAYS returns a tuple of (text, metadata_tuple) entries. If the Rust helper
+    fails or produces no chunks, return a single fallback chunk containing the
+    whole file text and minimal metadata.
+    """
     filepath = Path(filepath_str)
     chunks = []
 
     logger.info(f"🔧 Parsing Rust file: {filepath}")
 
-     # The binary built from `tools/src/main.rs` lives in
-    # `tools/src/target/release/rust_parser` – it is *always* shipped
-    # with the release.
     rust_helper = Path("./tools/src/target/release/rust_parser")
     if not rust_helper.is_file():
-        logger.warning(f"Rust parser binary not found: {rust_helper}. Skipping Rust file {filepath}")
-        return tuple()
-
-    # Always use the helper; if it is missing, the whole Rust parsing
-    # step will simply return an empty tuple, which is the safest
-    # fallback.
-    logger.info(f"🔧 Using Rust parser binary: {rust_helper}")
+        logger.warning(f"Rust parser binary not found: {rust_helper}. Using fallback whole-file chunk for {filepath}")
+        code = filepath.read_text(encoding="utf-8")
+        fallback_meta = {
+            "file": str(filepath),
+            "name": filepath.name,
+            "type": "file",
+            "line": 1,
+            "language": "rust",
+        }
+        return ((code, tuple(fallback_meta.items())),)
 
     try:
         code = filepath.read_text(encoding="utf-8")
         lines = code.splitlines(keepends=True)
 
-
-
-        # Run the Rust binary.
         proc = subprocess.run(
             [str(rust_helper), str(filepath)],
             capture_output=True,
@@ -1582,18 +1589,22 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
         for i, d in enumerate(decls):
             start_line = max(0, d.get("start_line", 1) - 1)
             end_line = d.get("end_line", start_line + 1)
+            # guard the end_line to be at least start_line+1
+            if end_line <= start_line:
+                end_line = start_line + 1
+
+            # Make slice safe even if end_line > len(lines)
             chunk_code = "".join(lines[start_line:end_line])
 
-            # Build comprehensive metadata
             metadata = {
                 "file": str(filepath),
-                "name": d.get("name", ""),
-                "type": d.get("type_", ""),
+                "name": d.get("name", "") or filepath.name,
+                "type": d.get("type_", "") or "unknown",
                 "line": start_line + 1,
                 "language": "rust",
             }
 
-            # Add Rust-specific metadata
+            # copy other fields defensively
             if d.get("visibility"):
                 metadata["visibility"] = d.get("visibility")
             if d.get("is_async"):
@@ -1613,24 +1624,21 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
             if d.get("parameters"):
                 metadata["parameters"] = d.get("parameters")
 
-            # Log what we found
             logger.info(f"    Declaration {i+1}: {metadata.get('type', 'unknown')} {metadata.get('name', 'unnamed')}")
 
-            chunk_text = f"File: {filepath}\nType: {d.get('type_')}\nName: {d.get('name')}\n"
-
-            # Add Rust-specific details to text
-            if d.get("visibility"):
-                chunk_text += f"Visibility: {d.get('visibility')}\n"
-            if d.get("is_async"):
+            chunk_text = f"File: {filepath}\nType: {metadata.get('type')}\nName: {metadata.get('name')}\n"
+            if metadata.get("visibility"):
+                chunk_text += f"Visibility: {metadata.get('visibility')}\n"
+            if metadata.get("is_async"):
                 chunk_text += "Async: yes\n"
-            if d.get("is_unsafe"):
+            if metadata.get("is_unsafe"):
                 chunk_text += "Unsafe: yes\n"
-            if d.get("generics"):
-                chunk_text += f"Generics: {d.get('generics')}\n"
-            if d.get("traits"):
-                chunk_text += f"Implements: {', '.join(d.get('traits', []))}\n"
-            if d.get("fields"):
-                chunk_text += f"Fields: {', '.join(d.get('fields', []))}\n"
+            if metadata.get("generics"):
+                chunk_text += f"Generics: {', '.join(metadata.get('generics', []))}\n"
+            if metadata.get("implements_traits"):
+                chunk_text += f"Implements: {', '.join(metadata.get('implements_traits', []))}\n"
+            if metadata.get("fields"):
+                chunk_text += f"Fields: {', '.join(metadata.get('fields', []))}\n"
 
             chunk_text += f"Code:\n{chunk_code}"
 
@@ -1639,18 +1647,29 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
                 "metadata": metadata
             })
 
-        if chunks:
-            logger.info(f"✅ Successfully parsed {len(chunks)} chunks from {filepath}")
-            return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks)
-        else:
-            logger.warning(f"❌ AST helper returned declarations but no chunks were created")
-
     except Exception as e:
         logger.warning(f"Rust AST helper failed for {filepath}: {e}")
 
-    final_result = tuple((c["text"], tuple(c["metadata"].items())) for c in chunks)
-    logger.info(f"📦 Final result: {len(final_result)} chunks with metadata")
-    return final_result
+    # If parser produced chunks, return them in the (text, metadata_tuple) format expected by parse_rust_file()
+    if chunks:
+        logger.info(f"✅ Successfully parsed {len(chunks)} chunks from {filepath}")
+        return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks)
+
+    # Fallback: emit a whole-file chunk to keep counts consistent
+    logger.warning(f"❌ No chunks created from Rust AST for {filepath}. Emitting fallback whole-file chunk.")
+    try:
+        full_code = filepath.read_text(encoding="utf-8")
+    except Exception:
+        full_code = ""
+    fallback_meta = {
+        "file": str(filepath),
+        "name": filepath.name,
+        "type": "file",
+        "line": 1,
+        "language": "rust",
+    }
+    return ((full_code, tuple(fallback_meta.items())),)
+
 
 @lru_cache(maxsize=1000)
 def parse_shell_file_cached(filepath_str: str) -> tuple:
@@ -1862,6 +1881,15 @@ def build_indexes():
     # Step 1: Parse codebase and build enhanced graph
     # -------------------------------------------------
     texts, metadatas = index_codebase(CODEBASE_PATH)
+    if len(texts) != len(metadatas):
+        # Helpful debug output to find offending files quickly
+        logger.error(
+            "Indexing mismatch: %d texts vs %d metadata entries. Aborting index build.",
+            len(texts),
+            len(metadatas),
+        )
+        # raise so CI/dev runs fail fast and you can inspect logs
+        raise ValueError(f"Indexing produced {len(texts)} texts but {len(metadatas)} metadata entries")
 
     logger.info("=== Sample metadata inspection ===")
     for i, m in enumerate(metadatas[:5]):  # Check first 5