reduce imports
This commit is contained in:
+102
-58
@@ -2505,7 +2505,7 @@ def parse_rust_file(filepath: Path) -> List[Dict]:
|
||||
|
||||
@lru_cache(maxsize=1000)
|
||||
def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
|
||||
"""Fixed Rust parsing that properly handles imports as metadata, not separate entities."""
|
||||
"""Fixed Rust parsing that aggregates imports and prioritizes code entities."""
|
||||
filepath = Path(filepath_str)
|
||||
chunks = []
|
||||
|
||||
@@ -2530,19 +2530,23 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
|
||||
decls = json.loads(proc.stdout)
|
||||
logger.info(f" AST helper found {len(decls)} declarations")
|
||||
|
||||
# Collect imports first to attach to actual code entities
|
||||
# Collect imports to attach to code entities, but don't create separate import chunks
|
||||
file_imports = []
|
||||
code_decls = []
|
||||
|
||||
for d in decls:
|
||||
if d.get("type_") == "import":
|
||||
file_imports.extend(d.get("parameters", []))
|
||||
# Collect import paths but don't create chunks for them
|
||||
import_paths = d.get("parameters", [])
|
||||
file_imports.extend(import_paths)
|
||||
logger.debug(f" Found import: {import_paths}")
|
||||
else:
|
||||
# Only process actual code entities
|
||||
code_decls.append(d)
|
||||
|
||||
logger.info(f" Found {len(file_imports)} imports and {len(code_decls)} code declarations")
|
||||
logger.info(f" Found {len(file_imports)} imports to attach to {len(code_decls)} code entities")
|
||||
|
||||
# Process actual code entities (functions, structs, etc.)
|
||||
# Process ACTUAL CODE entities (functions, structs, etc.)
|
||||
for i, d in enumerate(code_decls):
|
||||
start_line = max(1, d.get("start_line", 1))
|
||||
end_line = max(start_line, d.get("end_line", start_line + 1))
|
||||
@@ -2555,26 +2559,34 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
|
||||
|
||||
chunk_code = "".join(lines[start_line-1:end_line])
|
||||
|
||||
# Skip if we don't have meaningful content
|
||||
if not chunk_code.strip() or len(chunk_code.strip()) < 10:
|
||||
continue
|
||||
|
||||
metadata = {
|
||||
"file": str(filepath),
|
||||
"name": d.get("name", "") or filepath.stem,
|
||||
"type": d.get("type_", "") or "unknown",
|
||||
"line": start_line,
|
||||
"language": "rust",
|
||||
"imports": file_imports, # Attach imports to ALL code entities in this file
|
||||
"imports": file_imports, # Attach ALL file imports to each code entity
|
||||
}
|
||||
|
||||
# Enhanced metadata extraction for code entities
|
||||
enhanced_fields = [
|
||||
"visibility", "is_async", "is_unsafe", "generics", "traits",
|
||||
"fields", "methods", "return_type", "parameters", "docstring",
|
||||
"calls" # calls from the specific entity
|
||||
"calls"
|
||||
]
|
||||
|
||||
for field in enhanced_fields:
|
||||
if field in d and d[field] is not None:
|
||||
metadata[field] = d[field]
|
||||
|
||||
# Skip if it's just a module declaration without real content
|
||||
if metadata["type"] == "module" and not metadata.get("docstring") and len(chunk_code.strip()) < 20:
|
||||
continue
|
||||
|
||||
logger.info(f" Code {i+1}: {metadata.get('type', 'unknown')} {metadata.get('name', 'unnamed')}")
|
||||
|
||||
# Build rich chunk text for CODE entities
|
||||
@@ -2584,73 +2596,79 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
|
||||
"metadata": metadata
|
||||
})
|
||||
|
||||
# If we have no code declarations but have imports, create a file-level chunk
|
||||
if not chunks and file_imports:
|
||||
logger.info(" Creating file-level chunk for imports-only file")
|
||||
chunks.append(create_file_level_chunk(filepath, file_imports))
|
||||
# If we have no code declarations but have meaningful content, create file-level chunk
|
||||
if not chunks and len(code.strip()) > 100:
|
||||
logger.info(" Creating file-level chunk for content-rich file")
|
||||
chunks.append(create_file_level_chunk(filepath, file_imports, code))
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Rust AST helper failed for {filepath}: {e}")
|
||||
return create_fallback_chunk(filepath)
|
||||
|
||||
if chunks:
|
||||
logger.info(f"✅ Successfully parsed {len(chunks)} chunks from {filepath}")
|
||||
logger.info(f"✅ Successfully parsed {len(chunks)} code chunks from {filepath}")
|
||||
return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks)
|
||||
|
||||
logger.warning(f"❌ No chunks created from Rust AST for {filepath}. Emitting fallback.")
|
||||
logger.warning(f"❌ No meaningful code chunks from Rust AST for {filepath}. Emitting fallback.")
|
||||
return create_fallback_chunk(filepath)
|
||||
|
||||
def build_rust_code_chunk_text(metadata: dict, code: str) -> str:
|
||||
"""Build comprehensive Rust code chunk text."""
|
||||
"""Build comprehensive Rust code chunk text - focused on ACTUAL CODE."""
|
||||
parts = [
|
||||
f"File: {metadata.get('file', '')}",
|
||||
f"Type: {metadata.get('type', 'unknown')}",
|
||||
f"Name: {metadata.get('name', '')}",
|
||||
]
|
||||
|
||||
# Add enhanced metadata for CODE entities
|
||||
# Only include the most valuable metadata
|
||||
if metadata.get("visibility"):
|
||||
parts.append(f"Visibility: {metadata['visibility']}")
|
||||
if metadata.get("is_async"):
|
||||
parts.append("Async: yes")
|
||||
if metadata.get("is_unsafe"):
|
||||
parts.append("Unsafe: yes")
|
||||
if metadata.get("generics"):
|
||||
parts.append(f"Generics: {', '.join(metadata['generics'])}")
|
||||
if metadata.get("traits"):
|
||||
parts.append(f"Implements: {', '.join(metadata['traits'])}")
|
||||
if metadata.get("fields"):
|
||||
parts.append(f"Fields: {', '.join(metadata['fields'])}")
|
||||
if metadata.get("imports"):
|
||||
parts.append(f"Imports: {', '.join(metadata['imports'])}")
|
||||
if metadata.get("calls"):
|
||||
parts.append(f"Calls: {', '.join(metadata['calls'])}")
|
||||
if metadata.get("docstring"):
|
||||
parts.append(f"Doc: {metadata['docstring']}")
|
||||
# Only include substantial docstrings
|
||||
doc = metadata['docstring']
|
||||
if len(doc) > 20 and len(doc) < 300:
|
||||
parts.append(f"Doc: {doc}")
|
||||
|
||||
# Include imports and calls only if they exist and are meaningful
|
||||
meaningful_imports = [imp for imp in metadata.get("imports", [])
|
||||
if len(imp) > 3 and not imp.startswith(('c:', 'b:', 'p:', 'r:', 't:', 'v:'))]
|
||||
if meaningful_imports:
|
||||
parts.append(f"Imports: {', '.join(meaningful_imports[:5])}") # Limit to top 5
|
||||
|
||||
meaningful_calls = [call for call in metadata.get("calls", [])
|
||||
if len(call) > 2 and not call in ['if', 'for', 'let', 'mut', 'pub']]
|
||||
if meaningful_calls:
|
||||
parts.append(f"Calls: {', '.join(meaningful_calls[:5])}") # Limit to top 5
|
||||
|
||||
parts.append(f"Code:\n{code}")
|
||||
return "\n".join(parts)
|
||||
|
||||
def create_file_level_chunk(filepath: Path, imports: list) -> dict:
|
||||
"""Create a chunk for files that only have imports."""
|
||||
try:
|
||||
full_code = filepath.read_text(encoding="utf-8")
|
||||
except Exception:
|
||||
full_code = ""
|
||||
|
||||
def create_file_level_chunk(filepath: Path, imports: list, code: str) -> dict:
|
||||
"""Create a file-level chunk only for files with substantial content."""
|
||||
metadata = {
|
||||
"file": str(filepath),
|
||||
"name": filepath.stem,
|
||||
"type": "module", # Treat import-only files as modules
|
||||
"type": "file",
|
||||
"line": 1,
|
||||
"language": "rust",
|
||||
"imports": imports,
|
||||
}
|
||||
|
||||
chunk_text = f"File: {filepath}\nType: module\nName: {filepath.stem}\nImports: {', '.join(imports)}\nCode:\n{full_code}"
|
||||
# Only include meaningful imports
|
||||
meaningful_imports = [imp for imp in imports if len(imp) > 3]
|
||||
|
||||
chunk_text = f"File: {filepath}\nType: file\nName: {filepath.stem}"
|
||||
if meaningful_imports:
|
||||
chunk_text += f"\nImports: {', '.join(meaningful_imports[:3])}"
|
||||
chunk_text += f"\nCode:\n{code}"
|
||||
|
||||
return {"text": chunk_text, "metadata": metadata}
|
||||
|
||||
def create_fallback_chunk(filepath: Path) -> tuple:
|
||||
"""Create a minimal fallback chunk with proper metadata."""
|
||||
try:
|
||||
@@ -3573,44 +3591,70 @@ def build_indexes():
|
||||
# -------------------------------------------------
|
||||
|
||||
def _add_import_edges_enhanced(graph, file_node_id, importing_node_id, imports, lang):
|
||||
"""Enhanced import edge creation that resolves Rust paths to actual files."""
|
||||
"""Enhanced import edge creation that avoids symbolic node spam."""
|
||||
new_nodes = {}
|
||||
|
||||
for imp_path in imports:
|
||||
if not imp_path or not isinstance(imp_path, str):
|
||||
continue
|
||||
|
||||
# Skip standard library and external crates for now
|
||||
if any(imp_path.startswith(prefix) for prefix in ['std::', 'core::', 'alloc::', 'bevy::', 'serde::']):
|
||||
# Skip single-letter and obviously symbolic imports
|
||||
if len(imp_path.strip()) <= 1 or imp_path in ['c', 'b', 'p', 'r', 't', 'v', 'd', 'm', 'q', 's', 'n', 'f', 'a']:
|
||||
continue
|
||||
|
||||
# Try to resolve relative paths within the codebase
|
||||
# Skip common standard library and external crates (they create too much noise)
|
||||
external_prefixes = ['std::', 'core::', 'alloc::', 'bevy::', 'serde::', 'rayon::', 'rand::']
|
||||
if any(imp_path.startswith(prefix) for prefix in external_prefixes):
|
||||
# Only create ONE symbolic node for the entire crate, not individual items
|
||||
crate_name = imp_path.split('::')[0]
|
||||
if crate_name:
|
||||
symbolic_id = f"rust::external::{crate_name}"
|
||||
if symbolic_id not in graph.graph.nodes:
|
||||
graph.add_node(
|
||||
symbolic_id,
|
||||
type="external_crate",
|
||||
name=crate_name,
|
||||
file="",
|
||||
lang="rust",
|
||||
is_external=True
|
||||
)
|
||||
new_nodes[symbolic_id] = {
|
||||
"type": "external_crate",
|
||||
"name": crate_name,
|
||||
"file": "",
|
||||
"lang": "rust"
|
||||
}
|
||||
if not graph.graph.has_edge(importing_node_id, symbolic_id):
|
||||
graph.add_edge(importing_node_id, symbolic_id, "imports")
|
||||
continue
|
||||
|
||||
# For internal imports, try to resolve to actual files
|
||||
resolved_path = resolve_rust_import(imp_path, graph.root_path)
|
||||
if resolved_path:
|
||||
# Create edge to the actual file
|
||||
target_file_node_id = f"file::{resolved_path}"
|
||||
if not graph.graph.has_edge(importing_node_id, target_file_node_id):
|
||||
graph.add_edge(importing_node_id, target_file_node_id, "imports")
|
||||
else:
|
||||
# Create symbolic node for external/unresolved imports
|
||||
symbolic_id = f"rust::external::{imp_path}"
|
||||
if symbolic_id not in graph.graph.nodes:
|
||||
graph.add_node(
|
||||
symbolic_id,
|
||||
type="external",
|
||||
name=imp_path,
|
||||
file="",
|
||||
lang="rust",
|
||||
is_external=True
|
||||
)
|
||||
new_nodes[symbolic_id] = {
|
||||
"type": "external",
|
||||
"name": imp_path,
|
||||
"file": "",
|
||||
"lang": "rust"
|
||||
}
|
||||
if not graph.graph.has_edge(importing_node_id, symbolic_id):
|
||||
graph.add_edge(importing_node_id, symbolic_id, "imports")
|
||||
# Only create symbolic nodes for substantial internal paths
|
||||
if '::' in imp_path and len(imp_path) > 5:
|
||||
symbolic_id = f"rust::symbol::{imp_path}"
|
||||
if symbolic_id not in graph.graph.nodes:
|
||||
graph.add_node(
|
||||
symbolic_id,
|
||||
type="symbol",
|
||||
name=imp_path,
|
||||
file="",
|
||||
lang="rust",
|
||||
is_symbolic=True
|
||||
)
|
||||
new_nodes[symbolic_id] = {
|
||||
"type": "symbol",
|
||||
"name": imp_path,
|
||||
"file": "",
|
||||
"lang": "rust"
|
||||
}
|
||||
if not graph.graph.has_edge(importing_node_id, symbolic_id):
|
||||
graph.add_edge(importing_node_id, symbolic_id, "imports")
|
||||
|
||||
return new_nodes
|
||||
|
||||
|
||||
Reference in New Issue
Block a user