diff --git a/mcp_codebase.py b/mcp_codebase.py index c103f99..963c4ee 100644 --- a/mcp_codebase.py +++ b/mcp_codebase.py @@ -1635,290 +1635,13 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple: except Exception as e: logger.warning(f"Rust AST helper failed for {filepath}: {e}; falling back to regex") - - # --- Import extraction -------------------------------------------- - import_re = re.compile(r'^\s*use\s+([^;]+);', re.MULTILINE) - - # Fallback: Regex-based Rust parsing - # --- Extract Rust imports ---------------------------------------------- - imported_modules = import_re.findall(code) # ["std::fmt", "crate::pathfinding::PathNode", ...] - imports_clean = sorted({imp.split("::")[0] for imp in imported_modules}) - # ----------------------------------------------------------------------- - - # fallback: Regex-based Rust parsing - logger.info(f" Falling back to regex parsing for {filepath}") - regex_chunks = _parse_rust_with_regex(filepath, code, lines, import_re) - chunks.extend(regex_chunks) - logger.info(f" Regex found {len(regex_chunks)} chunks") - - # Attach the imports list to every chunk before returning - for c in chunks: - md = dict(c["metadata"]) - md["imports"] = imports_clean - c["metadata"] = md - - except Exception as e: logger.warning(f"parse_rust_file failed {filepath}: {e}") - # Very simple call extractor: finds foo(), bar::(), module::foo() - call_re = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)\s*!\s*\(|([A-Za-z_][A-Za-z0-9_:]*)\s*\(', re.MULTILINE) - calls = [] - - for match in call_re.findall(code): - name1, name2 = match - name = name1 or name2 - if "::" in name: - name = name.split("::")[-1] - # filter out keywords and type constructors - if name not in ["if", "for", "match", "while", "loop"]: - calls.append(name) - - # attach to every chunk belonging to the file - for c in chunks: - md = dict(c["metadata"]) - md["calls"] = calls - c["metadata"] = md - - final_result = tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) logger.info(f"📦 Final result: {len(final_result)} chunks with metadata") return final_result -def _parse_rust_with_regex(filepath: Path, code: str, lines: list, import_re) -> list: - """Fallback regex-based Rust parser.""" - chunks = [] - - # Pattern for Rust functions (including async, unsafe, methods) - func_pattern = re.compile( - r'^(?:pub\\s+)?(?:async\\s+)?(?:unsafe\\s+)?fn\\s+(\\w+)\\s*[\u003c(]', - re.MULTILINE - ) - - # Pattern for structs - struct_pattern = re.compile( - r'^(?:pub\s+)?struct\s+(\w+)\s*(?:<[^>]*>)?\s*\{', - re.MULTILINE - ) - - # Pattern for enums - enum_pattern = re.compile( - r'^(?:pub\s+)?enum\s+(\w+)\s*\{', - re.MULTILINE - ) - - # Pattern for traits - trait_pattern = re.compile( - r'^(?:pub\s+)?trait\s+(\w+)\s*\{', - re.MULTILINE - ) - - # Pattern for impl blocks - impl_pattern = re.compile( - r'^impl\s+(?:<[^>]*>)?\s*(\w+)\s*(?:<[^>]*>)?\s*\{', - re.MULTILINE - ) - - # Pattern for modules - mod_pattern = re.compile( - r'^(?:pub\s+)?mod\s+(\w+)\s*\{', - re.MULTILINE - ) - - # Parse functions - for match in func_pattern.finditer(code): - start_pos = match.start() - start_line = code[:start_pos].count('\\n') - end_line = _find_rust_brace_block_end(lines, start_line) - func_code = "".join(lines[start_line:end_line + 1]) - - # Extract function name - func_name = match.group(1) - - # Extract visibility (pub or private) - visibility = "pub" if "pub" in func_code else "private" - - # Detect async/unsafe - is_async = "async" in func_code - is_unsafe = "unsafe" in func_code - is_pub = 'pub' in match.group(0) - - chunk_text = f"File: {filepath}\nType: function\nName: {func_name}\n" - if is_async: - chunk_text += "Async: yes\n" - if is_unsafe: - chunk_text += "Unsafe: yes\n" - if is_pub: - chunk_text += "Visibility: pub\n" - chunk_text += f"Code:\n{func_code}" - - metadata = { - "file": str(filepath), - "name": func_name, - "type": "function", - "line": start_line + 1, - "language": "rust", - "is_async": is_async, - "is_unsafe": is_unsafe, - "visibility": "pub" if is_pub else "private" - } - - chunks.append({"text": chunk_text, "metadata": metadata}) - - # Parse structs - for match in struct_pattern.finditer(code): - struct_name = match.group(1) - start_pos = match.start() - start_line = code[:start_pos].count('\n') - end_line = _find_rust_brace_block_end(lines, start_line) - struct_code = "".join(lines[start_line:end_line + 1]) - - # Extract fields from struct - fields = _extract_rust_struct_fields(struct_code) - - chunk_text = f"File: {filepath}\nType: struct\nName: {struct_name}\n" - if fields: - chunk_text += f"Fields: {', '.join(fields)}\n" - chunk_text += f"Code:\n{struct_code}" - - metadata = { - "file": str(filepath), - "name": struct_name, - "type": "struct", - "line": start_line + 1, - "language": "rust", - "fields": fields - } - - chunks.append({"text": chunk_text, "metadata": metadata}) - - # Parse enums - for match in enum_pattern.finditer(code): - enum_name = match.group(1) - start_pos = match.start() - start_line = code[:start_pos].count('\n') - end_line = _find_rust_brace_block_end(lines, start_line) - enum_code = "".join(lines[start_line:end_line + 1]) - - # Extract variants from enum - variants = _extract_rust_enum_variants(enum_code) - - chunk_text = f"File: {filepath}\nType: enum\nName: {enum_name}\n" - if variants: - chunk_text += f"Variants: {', '.join(variants)}\n" - chunk_text += f"Code:\n{enum_code}" - - metadata = { - "file": str(filepath), - "name": enum_name, - "type": "enum", - "line": start_line + 1, - "language": "rust", - "variants": variants - } - - chunks.append({"text": chunk_text, "metadata": metadata}) - - # Parse traits - for match in trait_pattern.finditer(code): - trait_name = match.group(1) - start_pos = match.start() - start_line = code[:start_pos].count('\n') - end_line = _find_rust_brace_block_end(lines, start_line) - trait_code = "".join(lines[start_line:end_line + 1]) - - # Extract method signatures from trait - methods = _extract_rust_trait_methods(trait_code) - - chunk_text = f"File: {filepath}\nType: trait\nName: {trait_name}\n" - if methods: - chunk_text += f"Methods: {', '.join(methods)}\n" - chunk_text += f"Code:\n{trait_code}" - - metadata = { - "file": str(filepath), - "name": trait_name, - "type": "trait", - "line": start_line + 1, - "language": "rust", - "methods": methods - } - - chunks.append({"text": chunk_text, "metadata": metadata}) - - # Parse impl blocks - for match in impl_pattern.finditer(code): - impl_target = match.group(1) - start_pos = match.start() - start_line = code[:start_pos].count('\n') - end_line = _find_rust_brace_block_end(lines, start_line) - impl_code = "".join(lines[start_line:end_line + 1]) - - # Extract methods from impl block - impl_methods = _extract_rust_impl_methods(impl_code) - - chunk_text = f"File: {filepath}\nType: impl\nTarget: {impl_target}\n" - if impl_methods: - chunk_text += f"Implementation Methods: {', '.join(impl_methods)}\n" - chunk_text += f"Code:\n{impl_code}" - - metadata = { - "file": str(filepath), - "name": f"impl_{impl_target}", - "type": "impl", - "line": start_line + 1, - "language": "rust", - "target": impl_target, - "methods": impl_methods - } - - chunks.append({"text": chunk_text, "metadata": metadata}) - - return chunks - -def _find_rust_brace_block_end(lines, start_line): - """Find the closing brace for Rust code blocks.""" - brace_count = 0 - for i in range(start_line, len(lines)): - line = lines[i] - brace_count += line.count('{') - brace_count -= line.count('}') - if brace_count == 0: - return i - return len(lines) - 1 - -def _extract_rust_struct_fields(struct_code): - """Extract field names from Rust struct definition.""" - fields = [] - # Look for field patterns: field_name: Type, - field_matches = re.findall(r'(\w+)\s*:\s*[^,\n]+', struct_code) - fields.extend(field_matches) - return fields - -def _extract_rust_enum_variants(enum_code): - """Extract variant names from Rust enum definition.""" - variants = [] - # Look for variant patterns: VariantName, - variant_matches = re.findall(r'(\w+)(?:\([^)]*\))?\s*,', enum_code) - variants.extend(variant_matches) - return variants - -def _extract_rust_trait_methods(trait_code): - """Extract method names from Rust trait definition.""" - methods = [] - # Look for method signatures in trait - method_matches = re.findall(r'fn\s+(\w+)\s*\([^)]*\)', trait_code) - methods.extend(method_matches) - return methods - -def _extract_rust_impl_methods(impl_code): - """Extract method names from Rust impl block.""" - methods = [] - # Look for method implementations in impl block - method_matches = re.findall(r'fn\s+(\w+)\s*\([^)]*\)', impl_code) - methods.extend(method_matches) - return methods - @lru_cache(maxsize=1000) def parse_shell_file_cached(filepath_str: str) -> tuple: """Cached shell script parsing. Returns tuple for hashability."""