better rust parser
This commit is contained in:
+66
-63
@@ -2505,11 +2505,7 @@ def parse_rust_file(filepath: Path) -> List[Dict]:
|
||||
|
||||
@lru_cache(maxsize=1000)
|
||||
def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
|
||||
"""Cached Rust parsing with enhanced metadata for graph relationships.
|
||||
ALWAYS returns a tuple of (text, metadata_tuple) entries. If the Rust helper
|
||||
fails or produces no chunks, return a single fallback chunk containing the
|
||||
whole file text and minimal metadata.
|
||||
"""
|
||||
"""Enhanced Rust parsing with better metadata extraction."""
|
||||
filepath = Path(filepath_str)
|
||||
chunks = []
|
||||
|
||||
@@ -2518,15 +2514,7 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
|
||||
rust_helper = Path("./tools/target/release/parse_rust_ast")
|
||||
if not rust_helper.is_file():
|
||||
logger.warning(f"Rust parser binary not found: {rust_helper}. Using fallback whole-file chunk for {filepath}")
|
||||
code = filepath.read_text(encoding="utf-8")
|
||||
fallback_meta = {
|
||||
"file": str(filepath),
|
||||
"name": filepath.name,
|
||||
"type": "file",
|
||||
"line": 1,
|
||||
"language": "rust",
|
||||
}
|
||||
return ((code, tuple(fallback_meta.items())),)
|
||||
return create_fallback_chunk(filepath)
|
||||
|
||||
try:
|
||||
code = filepath.read_text(encoding="utf-8")
|
||||
@@ -2543,61 +2531,41 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
|
||||
logger.info(f" AST helper found {len(decls)} declarations")
|
||||
|
||||
for i, d in enumerate(decls):
|
||||
start_line = max(0, d.get("start_line", 1) - 1)
|
||||
end_line = d.get("end_line", start_line + 1)
|
||||
# guard the end_line to be at least start_line+1
|
||||
if end_line <= start_line:
|
||||
end_line = start_line + 1
|
||||
# FIX: Ensure valid line numbers
|
||||
start_line = max(1, d.get("start_line", 1))
|
||||
end_line = max(start_line, d.get("end_line", start_line + 1))
|
||||
|
||||
# Make slice safe even if end_line > len(lines)
|
||||
chunk_code = "".join(lines[start_line:end_line])
|
||||
# Ensure we don't exceed file bounds
|
||||
if start_line > len(lines):
|
||||
start_line = len(lines)
|
||||
if end_line > len(lines):
|
||||
end_line = len(lines)
|
||||
|
||||
chunk_code = "".join(lines[start_line-1:end_line])
|
||||
|
||||
metadata = {
|
||||
"file": str(filepath),
|
||||
"name": d.get("name", "") or filepath.name,
|
||||
"name": d.get("name", "") or filepath.stem,
|
||||
"type": d.get("type_", "") or "unknown",
|
||||
"line": start_line + 1,
|
||||
"line": start_line,
|
||||
"language": "rust",
|
||||
}
|
||||
|
||||
# copy other fields defensively
|
||||
if d.get("visibility"):
|
||||
metadata["visibility"] = d.get("visibility")
|
||||
if d.get("is_async"):
|
||||
metadata["is_async"] = True
|
||||
if d.get("is_unsafe"):
|
||||
metadata["is_unsafe"] = True
|
||||
if d.get("generics"):
|
||||
metadata["generics"] = d.get("generics")
|
||||
if d.get("traits"):
|
||||
metadata["implements_traits"] = d.get("traits")
|
||||
if d.get("fields"):
|
||||
metadata["fields"] = d.get("fields")
|
||||
if d.get("methods"):
|
||||
metadata["methods"] = d.get("methods")
|
||||
if d.get("return_type"):
|
||||
metadata["return_type"] = d.get("return_type")
|
||||
if d.get("parameters"):
|
||||
metadata["parameters"] = d.get("parameters")
|
||||
# Enhanced metadata extraction
|
||||
enhanced_fields = [
|
||||
"visibility", "is_async", "is_unsafe", "generics", "traits",
|
||||
"fields", "methods", "return_type", "parameters", "docstring",
|
||||
"imports", "calls" # NEW fields
|
||||
]
|
||||
|
||||
for field in enhanced_fields:
|
||||
if field in d and d[field] is not None:
|
||||
metadata[field] = d[field]
|
||||
|
||||
logger.info(f" Declaration {i+1}: {metadata.get('type', 'unknown')} {metadata.get('name', 'unnamed')}")
|
||||
|
||||
chunk_text = f"File: {filepath}\nType: {metadata.get('type')}\nName: {metadata.get('name')}\n"
|
||||
if metadata.get("visibility"):
|
||||
chunk_text += f"Visibility: {metadata.get('visibility')}\n"
|
||||
if metadata.get("is_async"):
|
||||
chunk_text += "Async: yes\n"
|
||||
if metadata.get("is_unsafe"):
|
||||
chunk_text += "Unsafe: yes\n"
|
||||
if metadata.get("generics"):
|
||||
chunk_text += f"Generics: {', '.join(metadata.get('generics', []))}\n"
|
||||
if metadata.get("implements_traits"):
|
||||
chunk_text += f"Implements: {', '.join(metadata.get('implements_traits', []))}\n"
|
||||
if metadata.get("fields"):
|
||||
chunk_text += f"Fields: {', '.join(metadata.get('fields', []))}\n"
|
||||
|
||||
chunk_text += f"Code:\n{chunk_code}"
|
||||
|
||||
# Build rich chunk text
|
||||
chunk_text = build_rust_chunk_text(metadata, chunk_code)
|
||||
chunks.append({
|
||||
"text": chunk_text,
|
||||
"metadata": metadata
|
||||
@@ -2605,27 +2573,61 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Rust AST helper failed for {filepath}: {e}")
|
||||
return create_fallback_chunk(filepath)
|
||||
|
||||
# If parser produced chunks, return them in the (text, metadata_tuple) format expected by parse_rust_file()
|
||||
if chunks:
|
||||
logger.info(f"✅ Successfully parsed {len(chunks)} chunks from {filepath}")
|
||||
return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks)
|
||||
|
||||
# Fallback: emit a whole-file chunk to keep counts consistent
|
||||
logger.warning(f"❌ No chunks created from Rust AST for {filepath}. Emitting fallback whole-file chunk.")
|
||||
logger.warning(f"❌ No chunks created from Rust AST for {filepath}. Emitting fallback.")
|
||||
return create_fallback_chunk(filepath)
|
||||
|
||||
def build_rust_chunk_text(metadata: dict, code: str) -> str:
|
||||
"""Build comprehensive Rust chunk text with all metadata."""
|
||||
parts = [
|
||||
f"File: {metadata.get('file', '')}",
|
||||
f"Type: {metadata.get('type', 'unknown')}",
|
||||
f"Name: {metadata.get('name', '')}",
|
||||
]
|
||||
|
||||
# Add enhanced metadata
|
||||
if metadata.get("visibility"):
|
||||
parts.append(f"Visibility: {metadata['visibility']}")
|
||||
if metadata.get("is_async"):
|
||||
parts.append("Async: yes")
|
||||
if metadata.get("is_unsafe"):
|
||||
parts.append("Unsafe: yes")
|
||||
if metadata.get("generics"):
|
||||
parts.append(f"Generics: {', '.join(metadata['generics'])}")
|
||||
if metadata.get("traits"):
|
||||
parts.append(f"Implements: {', '.join(metadata['traits'])}")
|
||||
if metadata.get("fields"):
|
||||
parts.append(f"Fields: {', '.join(metadata['fields'])}")
|
||||
if metadata.get("imports"):
|
||||
parts.append(f"Imports: {', '.join(metadata['imports'])}")
|
||||
if metadata.get("calls"):
|
||||
parts.append(f"Calls: {', '.join(metadata['calls'])}")
|
||||
|
||||
parts.append(f"Code:\n{code}")
|
||||
return "\n".join(parts)
|
||||
|
||||
def create_fallback_chunk(filepath: Path) -> tuple:
|
||||
"""Create a minimal fallback chunk with proper metadata."""
|
||||
try:
|
||||
full_code = filepath.read_text(encoding="utf-8")
|
||||
except Exception:
|
||||
full_code = ""
|
||||
|
||||
fallback_meta = {
|
||||
"file": str(filepath),
|
||||
"name": filepath.name,
|
||||
"name": filepath.stem, # Use stem instead of full name
|
||||
"type": "file",
|
||||
"line": 1,
|
||||
"language": "rust",
|
||||
}
|
||||
return ((full_code, tuple(fallback_meta.items())),)
|
||||
chunk_text = f"File: {filepath}\nType: file\nName: {filepath.stem}\nCode:\n{full_code}"
|
||||
|
||||
return ((chunk_text, tuple(fallback_meta.items())),)
|
||||
|
||||
@lru_cache(maxsize=1000)
|
||||
def parse_shell_file_cached(filepath_str: str) -> tuple:
|
||||
@@ -5445,6 +5447,7 @@ def signal_handler(sig, frame):
|
||||
parse_sql_file_cached.cache_clear()
|
||||
parse_go_file_cached.cache_clear()
|
||||
parse_java_file_cached.cache_clear()
|
||||
parse_rust_file_cached.cache_clear()
|
||||
parse_svelte_file_cached.cache_clear()
|
||||
parse_shell_file_cached.cache_clear()
|
||||
logger.info("✓ Caches cleared")
|
||||
|
||||
Reference in New Issue
Block a user