cleanup
This commit is contained in:
+88
-161
@@ -1547,96 +1547,108 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
|
||||
|
||||
logger.info(f"🔧 Parsing Rust file: {filepath}")
|
||||
|
||||
# The binary built from `tools/src/main.rs` lives in
|
||||
# `tools/src/target/release/rust_parser` – it is *always* shipped
|
||||
# with the release.
|
||||
rust_helper = Path("./tools/src/target/release/rust_parser")
|
||||
|
||||
# Always use the helper; if it is missing, the whole Rust parsing
|
||||
# step will simply return an empty tuple, which is the safest
|
||||
# fallback.
|
||||
logger.info(f"🔧 Using Rust parser binary: {rust_helper}")
|
||||
|
||||
try:
|
||||
code = filepath.read_text(encoding="utf-8")
|
||||
lines = code.splitlines(keepends=True)
|
||||
|
||||
# Try using rust-analyzer AST if available
|
||||
rust_helper = Path("./tools/parse_rust_ast")
|
||||
if rust_helper.exists():
|
||||
logger.info(f" Using Rust AST helper: {rust_helper}")
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[str(rust_helper), str(filepath)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=20
|
||||
)
|
||||
decls = json.loads(proc.stdout)
|
||||
logger.info(f" AST helper found {len(decls)} declarations")
|
||||
# Capture the import paths for each file – they are specific to the file
|
||||
# and will be attached to every chunk emitted for this file.
|
||||
import_paths = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ItemUse):
|
||||
import_paths.append(use_tree_to_string(node.tree))
|
||||
|
||||
for i, d in enumerate(decls):
|
||||
start_line = max(0, d.get("start_line", 1) - 1)
|
||||
end_line = d.get("end_line", start_line + 1)
|
||||
chunk_code = "".join(lines[start_line:end_line])
|
||||
metadata["imports"] = import_paths
|
||||
|
||||
# Build comprehensive metadata
|
||||
metadata = {
|
||||
"file": str(filepath),
|
||||
"name": d.get("name", ""),
|
||||
"type": d.get("type_", ""),
|
||||
"line": start_line + 1,
|
||||
"language": "rust",
|
||||
}
|
||||
# Run the Rust binary.
|
||||
proc = subprocess.run(
|
||||
[str(rust_helper), str(filepath)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=20
|
||||
)
|
||||
decls = json.loads(proc.stdout)
|
||||
logger.info(f" AST helper found {len(decls)} declarations")
|
||||
|
||||
# Add Rust-specific metadata
|
||||
if d.get("visibility"):
|
||||
metadata["visibility"] = d.get("visibility")
|
||||
if d.get("is_async"):
|
||||
metadata["is_async"] = True
|
||||
if d.get("is_unsafe"):
|
||||
metadata["is_unsafe"] = True
|
||||
if d.get("generics"):
|
||||
metadata["generics"] = d.get("generics")
|
||||
if d.get("traits"):
|
||||
metadata["implements_traits"] = d.get("traits")
|
||||
if d.get("fields"):
|
||||
metadata["fields"] = d.get("fields")
|
||||
if d.get("methods"):
|
||||
metadata["methods"] = d.get("methods")
|
||||
if d.get("return_type"):
|
||||
metadata["return_type"] = d.get("return_type")
|
||||
if d.get("parameters"):
|
||||
metadata["parameters"] = d.get("parameters")
|
||||
for i, d in enumerate(decls):
|
||||
start_line = max(0, d.get("start_line", 1) - 1)
|
||||
end_line = d.get("end_line", start_line + 1)
|
||||
chunk_code = "".join(lines[start_line:end_line])
|
||||
|
||||
# Log what we found
|
||||
logger.info(f" Declaration {i+1}: {metadata.get('type', 'unknown')} {metadata.get('name', 'unnamed')}")
|
||||
# Build comprehensive metadata
|
||||
metadata = {
|
||||
"file": str(filepath),
|
||||
"name": d.get("name", ""),
|
||||
"type": d.get("type_", ""),
|
||||
"line": start_line + 1,
|
||||
"language": "rust",
|
||||
}
|
||||
|
||||
chunk_text = f"File: {filepath}\nType: {d.get('type_')}\nName: {d.get('name')}\n"
|
||||
# Add Rust-specific metadata
|
||||
if d.get("visibility"):
|
||||
metadata["visibility"] = d.get("visibility")
|
||||
if d.get("is_async"):
|
||||
metadata["is_async"] = True
|
||||
if d.get("is_unsafe"):
|
||||
metadata["is_unsafe"] = True
|
||||
if d.get("generics"):
|
||||
metadata["generics"] = d.get("generics")
|
||||
if d.get("traits"):
|
||||
metadata["implements_traits"] = d.get("traits")
|
||||
if d.get("fields"):
|
||||
metadata["fields"] = d.get("fields")
|
||||
if d.get("methods"):
|
||||
metadata["methods"] = d.get("methods")
|
||||
if d.get("return_type"):
|
||||
metadata["return_type"] = d.get("return_type")
|
||||
if d.get("parameters"):
|
||||
metadata["parameters"] = d.get("parameters")
|
||||
|
||||
# Add Rust-specific details to text
|
||||
if d.get("visibility"):
|
||||
chunk_text += f"Visibility: {d.get('visibility')}\n"
|
||||
if d.get("is_async"):
|
||||
chunk_text += "Async: yes\n"
|
||||
if d.get("is_unsafe"):
|
||||
chunk_text += "Unsafe: yes\n"
|
||||
if d.get("generics"):
|
||||
chunk_text += f"Generics: {d.get('generics')}\n"
|
||||
if d.get("traits"):
|
||||
chunk_text += f"Implements: {', '.join(d.get('traits', []))}\n"
|
||||
if d.get("fields"):
|
||||
chunk_text += f"Fields: {', '.join(d.get('fields', []))}\n"
|
||||
# Log what we found
|
||||
logger.info(f" Declaration {i+1}: {metadata.get('type', 'unknown')} {metadata.get('name', 'unnamed')}")
|
||||
|
||||
chunk_text += f"Code:\n{chunk_code}"
|
||||
chunk_text = f"File: {filepath}\nType: {d.get('type_')}\nName: {d.get('name')}\n"
|
||||
|
||||
chunks.append({
|
||||
"text": chunk_text,
|
||||
"metadata": metadata
|
||||
})
|
||||
# Add Rust-specific details to text
|
||||
if d.get("visibility"):
|
||||
chunk_text += f"Visibility: {d.get('visibility')}\n"
|
||||
if d.get("is_async"):
|
||||
chunk_text += "Async: yes\n"
|
||||
if d.get("is_unsafe"):
|
||||
chunk_text += "Unsafe: yes\n"
|
||||
if d.get("generics"):
|
||||
chunk_text += f"Generics: {d.get('generics')}\n"
|
||||
if d.get("traits"):
|
||||
chunk_text += f"Implements: {', '.join(d.get('traits', []))}\n"
|
||||
if d.get("fields"):
|
||||
chunk_text += f"Fields: {', '.join(d.get('fields', []))}\n"
|
||||
|
||||
if chunks:
|
||||
logger.info(f"✅ Successfully parsed {len(chunks)} chunks from {filepath}")
|
||||
return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks)
|
||||
else:
|
||||
logger.warning(f"❌ AST helper returned declarations but no chunks were created")
|
||||
chunk_text += f"Code:\n{chunk_code}"
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Rust AST helper failed for {filepath}: {e}; falling back to regex")
|
||||
chunks.append({
|
||||
"text": chunk_text,
|
||||
"metadata": metadata
|
||||
})
|
||||
|
||||
if chunks:
|
||||
logger.info(f"✅ Successfully parsed {len(chunks)} chunks from {filepath}")
|
||||
return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks)
|
||||
else:
|
||||
logger.warning(f"❌ AST helper returned declarations but no chunks were created")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"parse_rust_file failed {filepath}: {e}")
|
||||
logger.warning(f"Rust AST helper failed for {filepath}: {e}")
|
||||
|
||||
final_result = tuple((c["text"], tuple(c["metadata"].items())) for c in chunks)
|
||||
logger.info(f"📦 Final result: {len(final_result)} chunks with metadata")
|
||||
@@ -2044,27 +2056,6 @@ def build_indexes():
|
||||
created_nodes[node_id]["implements"] = m.get("implements", [])
|
||||
created_nodes[node_id]["fields"] = m.get("fields", [])
|
||||
created_nodes[node_id]["variants"] = m.get("variants", [])
|
||||
|
||||
elif node_type == "impl":
|
||||
# Implementation blocks
|
||||
target = m.get("target", "unknown")
|
||||
node_id = f"{lang}::{node_type}::{fpath}::{target}"
|
||||
|
||||
graph.add_node(node_id, **base_attrs, type=node_type,
|
||||
target=target,
|
||||
traits=m.get("traits", []),
|
||||
methods=m.get("methods", []))
|
||||
|
||||
# Connect impl to file
|
||||
graph.add_edge(file_node_id, node_id, "contains")
|
||||
|
||||
# Store for second-pass
|
||||
created_nodes[node_id] = base_attrs.copy()
|
||||
created_nodes[node_id]["type"] = node_type
|
||||
created_nodes[node_id]["target"] = target
|
||||
created_nodes[node_id]["traits"] = m.get("traits", [])
|
||||
created_nodes[node_id]["methods"] = m.get("methods", [])
|
||||
|
||||
elif node_type == "trait":
|
||||
# Traits
|
||||
graph.add_node(node_id, **base_attrs, type=node_type,
|
||||
@@ -2094,37 +2085,7 @@ def build_indexes():
|
||||
else:
|
||||
imports = imports_raw
|
||||
|
||||
for imp in imports:
|
||||
# Try to find an existing node for this import
|
||||
candidate_id = None
|
||||
for node_id, data in created_nodes.items():
|
||||
if data.get("type") in {"module", "import"} and data.get("name") == imp:
|
||||
candidate_id = node_id
|
||||
break
|
||||
|
||||
if not candidate_id:
|
||||
# No existing node found - create a placeholder
|
||||
candidate_id = f"import::{imp}"
|
||||
if candidate_id not in graph.graph.nodes:
|
||||
# Double-check for a module node with this name
|
||||
found = False
|
||||
for n_id, n_data in created_nodes.items():
|
||||
if n_data.get("name") == imp and n_data.get("type") == "module":
|
||||
candidate_id = n_id
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
# Create a generic import node
|
||||
graph.add_node(
|
||||
candidate_id,
|
||||
type="Import",
|
||||
name=imp,
|
||||
lang="unknown",
|
||||
file="",
|
||||
)
|
||||
graph.add_edge(file_node_id, candidate_id, "imports")
|
||||
# Create the connection
|
||||
graph.add_edge(file_node_id, candidate_id, "imports")
|
||||
graph._add_import_edges(file_node_id, imports)
|
||||
|
||||
# ---------- FUNCTION CALLS ----------
|
||||
calls_raw = m.get("calls", [])
|
||||
@@ -2134,40 +2095,7 @@ def build_indexes():
|
||||
else:
|
||||
calls = calls_raw
|
||||
|
||||
for call in calls:
|
||||
# Try to find the function/method being called
|
||||
target_id = None
|
||||
for node_id, data in created_nodes.items():
|
||||
if data.get("type") in {"function", "method", "constructor"} and data.get("name") == call:
|
||||
target_id = node_id
|
||||
break
|
||||
|
||||
if not target_id:
|
||||
# No existing function found - create a placeholder
|
||||
target_id = f"call::{call}"
|
||||
if target_id not in graph.graph.nodes:
|
||||
# Double-check for a matching function
|
||||
found = False
|
||||
for n_id, n_data in created_nodes.items():
|
||||
if (
|
||||
n_data.get("name") == call
|
||||
and n_data.get("type") in {"function", "method", "constructor"}
|
||||
):
|
||||
target_id = n_id
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
# Create a generic function call node
|
||||
graph.add_node(
|
||||
target_id,
|
||||
type="FunctionCall",
|
||||
name=call,
|
||||
lang="unknown",
|
||||
file="",
|
||||
)
|
||||
graph.add_edge(node_id, target_id, "calls")
|
||||
# Create the connection
|
||||
graph.add_edge(node_id, target_id, "calls")
|
||||
graph._add_call_edges(node_id, calls, fpath)
|
||||
|
||||
|
||||
# -------------------------------------------------
|
||||
@@ -3732,7 +3660,6 @@ if __name__ == "__main__":
|
||||
try:
|
||||
logger.info("="*60)
|
||||
logger.info("🚀 MCP RAG Server is ready!")
|
||||
logger.info("Use 'python serve_http.py' for HTTP server")
|
||||
logger.info("Press Ctrl+C to stop")
|
||||
|
||||
# Just run in stdio mode by default
|
||||
|
||||
Reference in New Issue
Block a user