From 1b702cfb922ccf3dd0ac4da1fb3eeb0f8875801f Mon Sep 17 00:00:00 2001 From: Stephen Adamson Date: Sat, 15 Nov 2025 01:33:40 +0000 Subject: [PATCH] re-do rust --- mcp_codebase.py | 110 ++++++++++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 41 deletions(-) diff --git a/mcp_codebase.py b/mcp_codebase.py index a8d67f7..6e13bb0 100644 --- a/mcp_codebase.py +++ b/mcp_codebase.py @@ -290,6 +290,12 @@ def apply_contextual_weights_to_embeddings(embeddings_list: List[List[float]], """ weighted_embeddings = [] + if len(embeddings_list) != len(metadata_list): + raise ValueError( + f"Embedding/metadata mismatch: {len(embeddings_list)} embeddings vs {len(metadata_list)} metadata. " + "This will break downstream systems (Chroma)." + ) + for embedding, metadata in zip(embeddings_list, metadata_list): weight = calculate_contextual_weight(metadata) @@ -1543,32 +1549,33 @@ def parse_rust_file(filepath: Path) -> List[Dict]: @lru_cache(maxsize=1000) def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple: - """Cached Rust parsing with enhanced metadata for graph relationships.""" + """Cached Rust parsing with enhanced metadata for graph relationships. + ALWAYS returns a tuple of (text, metadata_tuple) entries. If the Rust helper + fails or produces no chunks, return a single fallback chunk containing the + whole file text and minimal metadata. + """ filepath = Path(filepath_str) chunks = [] logger.info(f"🔧 Parsing Rust file: {filepath}") - # The binary built from `tools/src/main.rs` lives in - # `tools/src/target/release/rust_parser` – it is *always* shipped - # with the release. rust_helper = Path("./tools/src/target/release/rust_parser") if not rust_helper.is_file(): - logger.warning(f"Rust parser binary not found: {rust_helper}. Skipping Rust file {filepath}") - return tuple() - - # Always use the helper; if it is missing, the whole Rust parsing - # step will simply return an empty tuple, which is the safest - # fallback. - logger.info(f"🔧 Using Rust parser binary: {rust_helper}") + logger.warning(f"Rust parser binary not found: {rust_helper}. Using fallback whole-file chunk for {filepath}") + code = filepath.read_text(encoding="utf-8") + fallback_meta = { + "file": str(filepath), + "name": filepath.name, + "type": "file", + "line": 1, + "language": "rust", + } + return ((code, tuple(fallback_meta.items())),) try: code = filepath.read_text(encoding="utf-8") lines = code.splitlines(keepends=True) - - - # Run the Rust binary. proc = subprocess.run( [str(rust_helper), str(filepath)], capture_output=True, @@ -1582,18 +1589,22 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple: for i, d in enumerate(decls): start_line = max(0, d.get("start_line", 1) - 1) end_line = d.get("end_line", start_line + 1) + # guard the end_line to be at least start_line+1 + if end_line <= start_line: + end_line = start_line + 1 + + # Make slice safe even if end_line > len(lines) chunk_code = "".join(lines[start_line:end_line]) - # Build comprehensive metadata metadata = { "file": str(filepath), - "name": d.get("name", ""), - "type": d.get("type_", ""), + "name": d.get("name", "") or filepath.name, + "type": d.get("type_", "") or "unknown", "line": start_line + 1, "language": "rust", } - # Add Rust-specific metadata + # copy other fields defensively if d.get("visibility"): metadata["visibility"] = d.get("visibility") if d.get("is_async"): @@ -1613,24 +1624,21 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple: if d.get("parameters"): metadata["parameters"] = d.get("parameters") - # Log what we found logger.info(f" Declaration {i+1}: {metadata.get('type', 'unknown')} {metadata.get('name', 'unnamed')}") - chunk_text = f"File: {filepath}\nType: {d.get('type_')}\nName: {d.get('name')}\n" - - # Add Rust-specific details to text - if d.get("visibility"): - chunk_text += f"Visibility: {d.get('visibility')}\n" - if d.get("is_async"): + chunk_text = f"File: {filepath}\nType: {metadata.get('type')}\nName: {metadata.get('name')}\n" + if metadata.get("visibility"): + chunk_text += f"Visibility: {metadata.get('visibility')}\n" + if metadata.get("is_async"): chunk_text += "Async: yes\n" - if d.get("is_unsafe"): + if metadata.get("is_unsafe"): chunk_text += "Unsafe: yes\n" - if d.get("generics"): - chunk_text += f"Generics: {d.get('generics')}\n" - if d.get("traits"): - chunk_text += f"Implements: {', '.join(d.get('traits', []))}\n" - if d.get("fields"): - chunk_text += f"Fields: {', '.join(d.get('fields', []))}\n" + if metadata.get("generics"): + chunk_text += f"Generics: {', '.join(metadata.get('generics', []))}\n" + if metadata.get("implements_traits"): + chunk_text += f"Implements: {', '.join(metadata.get('implements_traits', []))}\n" + if metadata.get("fields"): + chunk_text += f"Fields: {', '.join(metadata.get('fields', []))}\n" chunk_text += f"Code:\n{chunk_code}" @@ -1639,18 +1647,29 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple: "metadata": metadata }) - if chunks: - logger.info(f"✅ Successfully parsed {len(chunks)} chunks from {filepath}") - return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) - else: - logger.warning(f"❌ AST helper returned declarations but no chunks were created") - except Exception as e: logger.warning(f"Rust AST helper failed for {filepath}: {e}") - final_result = tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) - logger.info(f"📦 Final result: {len(final_result)} chunks with metadata") - return final_result + # If parser produced chunks, return them in the (text, metadata_tuple) format expected by parse_rust_file() + if chunks: + logger.info(f"✅ Successfully parsed {len(chunks)} chunks from {filepath}") + return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) + + # Fallback: emit a whole-file chunk to keep counts consistent + logger.warning(f"❌ No chunks created from Rust AST for {filepath}. Emitting fallback whole-file chunk.") + try: + full_code = filepath.read_text(encoding="utf-8") + except Exception: + full_code = "" + fallback_meta = { + "file": str(filepath), + "name": filepath.name, + "type": "file", + "line": 1, + "language": "rust", + } + return ((full_code, tuple(fallback_meta.items())),) + @lru_cache(maxsize=1000) def parse_shell_file_cached(filepath_str: str) -> tuple: @@ -1862,6 +1881,15 @@ def build_indexes(): # Step 1: Parse codebase and build enhanced graph # ------------------------------------------------- texts, metadatas = index_codebase(CODEBASE_PATH) + if len(texts) != len(metadatas): + # Helpful debug output to find offending files quickly + logger.error( + "Indexing mismatch: %d texts vs %d metadata entries. Aborting index build.", + len(texts), + len(metadatas), + ) + # raise so CI/dev runs fail fast and you can inspect logs + raise ValueError(f"Indexing produced {len(texts)} texts but {len(metadatas)} metadata entries") logger.info("=== Sample metadata inspection ===") for i, m in enumerate(metadatas[:5]): # Check first 5