remove rust regex parse
This commit is contained in:
-277
@@ -1635,290 +1635,13 @@ def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Rust AST helper failed for {filepath}: {e}; falling back to regex")
|
logger.warning(f"Rust AST helper failed for {filepath}: {e}; falling back to regex")
|
||||||
|
|
||||||
|
|
||||||
# --- Import extraction --------------------------------------------
|
|
||||||
import_re = re.compile(r'^\s*use\s+([^;]+);', re.MULTILINE)
|
|
||||||
|
|
||||||
# Fallback: Regex-based Rust parsing
|
|
||||||
# --- Extract Rust imports ----------------------------------------------
|
|
||||||
imported_modules = import_re.findall(code) # ["std::fmt", "crate::pathfinding::PathNode", ...]
|
|
||||||
imports_clean = sorted({imp.split("::")[0] for imp in imported_modules})
|
|
||||||
# -----------------------------------------------------------------------
|
|
||||||
|
|
||||||
# fallback: Regex-based Rust parsing
|
|
||||||
logger.info(f" Falling back to regex parsing for {filepath}")
|
|
||||||
regex_chunks = _parse_rust_with_regex(filepath, code, lines, import_re)
|
|
||||||
chunks.extend(regex_chunks)
|
|
||||||
logger.info(f" Regex found {len(regex_chunks)} chunks")
|
|
||||||
|
|
||||||
# Attach the imports list to every chunk before returning
|
|
||||||
for c in chunks:
|
|
||||||
md = dict(c["metadata"])
|
|
||||||
md["imports"] = imports_clean
|
|
||||||
c["metadata"] = md
|
|
||||||
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"parse_rust_file failed {filepath}: {e}")
|
logger.warning(f"parse_rust_file failed {filepath}: {e}")
|
||||||
|
|
||||||
# Very simple call extractor: finds foo(), bar::<T>(), module::foo()
|
|
||||||
call_re = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)\s*!\s*\(|([A-Za-z_][A-Za-z0-9_:]*)\s*\(', re.MULTILINE)
|
|
||||||
calls = []
|
|
||||||
|
|
||||||
for match in call_re.findall(code):
|
|
||||||
name1, name2 = match
|
|
||||||
name = name1 or name2
|
|
||||||
if "::" in name:
|
|
||||||
name = name.split("::")[-1]
|
|
||||||
# filter out keywords and type constructors
|
|
||||||
if name not in ["if", "for", "match", "while", "loop"]:
|
|
||||||
calls.append(name)
|
|
||||||
|
|
||||||
# attach to every chunk belonging to the file
|
|
||||||
for c in chunks:
|
|
||||||
md = dict(c["metadata"])
|
|
||||||
md["calls"] = calls
|
|
||||||
c["metadata"] = md
|
|
||||||
|
|
||||||
|
|
||||||
final_result = tuple((c["text"], tuple(c["metadata"].items())) for c in chunks)
|
final_result = tuple((c["text"], tuple(c["metadata"].items())) for c in chunks)
|
||||||
logger.info(f"📦 Final result: {len(final_result)} chunks with metadata")
|
logger.info(f"📦 Final result: {len(final_result)} chunks with metadata")
|
||||||
return final_result
|
return final_result
|
||||||
|
|
||||||
def _parse_rust_with_regex(filepath: Path, code: str, lines: list, import_re) -> list:
|
|
||||||
"""Fallback regex-based Rust parser."""
|
|
||||||
chunks = []
|
|
||||||
|
|
||||||
# Pattern for Rust functions (including async, unsafe, methods)
|
|
||||||
func_pattern = re.compile(
|
|
||||||
r'^(?:pub\\s+)?(?:async\\s+)?(?:unsafe\\s+)?fn\\s+(\\w+)\\s*[\u003c(]',
|
|
||||||
re.MULTILINE
|
|
||||||
)
|
|
||||||
|
|
||||||
# Pattern for structs
|
|
||||||
struct_pattern = re.compile(
|
|
||||||
r'^(?:pub\s+)?struct\s+(\w+)\s*(?:<[^>]*>)?\s*\{',
|
|
||||||
re.MULTILINE
|
|
||||||
)
|
|
||||||
|
|
||||||
# Pattern for enums
|
|
||||||
enum_pattern = re.compile(
|
|
||||||
r'^(?:pub\s+)?enum\s+(\w+)\s*\{',
|
|
||||||
re.MULTILINE
|
|
||||||
)
|
|
||||||
|
|
||||||
# Pattern for traits
|
|
||||||
trait_pattern = re.compile(
|
|
||||||
r'^(?:pub\s+)?trait\s+(\w+)\s*\{',
|
|
||||||
re.MULTILINE
|
|
||||||
)
|
|
||||||
|
|
||||||
# Pattern for impl blocks
|
|
||||||
impl_pattern = re.compile(
|
|
||||||
r'^impl\s+(?:<[^>]*>)?\s*(\w+)\s*(?:<[^>]*>)?\s*\{',
|
|
||||||
re.MULTILINE
|
|
||||||
)
|
|
||||||
|
|
||||||
# Pattern for modules
|
|
||||||
mod_pattern = re.compile(
|
|
||||||
r'^(?:pub\s+)?mod\s+(\w+)\s*\{',
|
|
||||||
re.MULTILINE
|
|
||||||
)
|
|
||||||
|
|
||||||
# Parse functions
|
|
||||||
for match in func_pattern.finditer(code):
|
|
||||||
start_pos = match.start()
|
|
||||||
start_line = code[:start_pos].count('\\n')
|
|
||||||
end_line = _find_rust_brace_block_end(lines, start_line)
|
|
||||||
func_code = "".join(lines[start_line:end_line + 1])
|
|
||||||
|
|
||||||
# Extract function name
|
|
||||||
func_name = match.group(1)
|
|
||||||
|
|
||||||
# Extract visibility (pub or private)
|
|
||||||
visibility = "pub" if "pub" in func_code else "private"
|
|
||||||
|
|
||||||
# Detect async/unsafe
|
|
||||||
is_async = "async" in func_code
|
|
||||||
is_unsafe = "unsafe" in func_code
|
|
||||||
is_pub = 'pub' in match.group(0)
|
|
||||||
|
|
||||||
chunk_text = f"File: {filepath}\nType: function\nName: {func_name}\n"
|
|
||||||
if is_async:
|
|
||||||
chunk_text += "Async: yes\n"
|
|
||||||
if is_unsafe:
|
|
||||||
chunk_text += "Unsafe: yes\n"
|
|
||||||
if is_pub:
|
|
||||||
chunk_text += "Visibility: pub\n"
|
|
||||||
chunk_text += f"Code:\n{func_code}"
|
|
||||||
|
|
||||||
metadata = {
|
|
||||||
"file": str(filepath),
|
|
||||||
"name": func_name,
|
|
||||||
"type": "function",
|
|
||||||
"line": start_line + 1,
|
|
||||||
"language": "rust",
|
|
||||||
"is_async": is_async,
|
|
||||||
"is_unsafe": is_unsafe,
|
|
||||||
"visibility": "pub" if is_pub else "private"
|
|
||||||
}
|
|
||||||
|
|
||||||
chunks.append({"text": chunk_text, "metadata": metadata})
|
|
||||||
|
|
||||||
# Parse structs
|
|
||||||
for match in struct_pattern.finditer(code):
|
|
||||||
struct_name = match.group(1)
|
|
||||||
start_pos = match.start()
|
|
||||||
start_line = code[:start_pos].count('\n')
|
|
||||||
end_line = _find_rust_brace_block_end(lines, start_line)
|
|
||||||
struct_code = "".join(lines[start_line:end_line + 1])
|
|
||||||
|
|
||||||
# Extract fields from struct
|
|
||||||
fields = _extract_rust_struct_fields(struct_code)
|
|
||||||
|
|
||||||
chunk_text = f"File: {filepath}\nType: struct\nName: {struct_name}\n"
|
|
||||||
if fields:
|
|
||||||
chunk_text += f"Fields: {', '.join(fields)}\n"
|
|
||||||
chunk_text += f"Code:\n{struct_code}"
|
|
||||||
|
|
||||||
metadata = {
|
|
||||||
"file": str(filepath),
|
|
||||||
"name": struct_name,
|
|
||||||
"type": "struct",
|
|
||||||
"line": start_line + 1,
|
|
||||||
"language": "rust",
|
|
||||||
"fields": fields
|
|
||||||
}
|
|
||||||
|
|
||||||
chunks.append({"text": chunk_text, "metadata": metadata})
|
|
||||||
|
|
||||||
# Parse enums
|
|
||||||
for match in enum_pattern.finditer(code):
|
|
||||||
enum_name = match.group(1)
|
|
||||||
start_pos = match.start()
|
|
||||||
start_line = code[:start_pos].count('\n')
|
|
||||||
end_line = _find_rust_brace_block_end(lines, start_line)
|
|
||||||
enum_code = "".join(lines[start_line:end_line + 1])
|
|
||||||
|
|
||||||
# Extract variants from enum
|
|
||||||
variants = _extract_rust_enum_variants(enum_code)
|
|
||||||
|
|
||||||
chunk_text = f"File: {filepath}\nType: enum\nName: {enum_name}\n"
|
|
||||||
if variants:
|
|
||||||
chunk_text += f"Variants: {', '.join(variants)}\n"
|
|
||||||
chunk_text += f"Code:\n{enum_code}"
|
|
||||||
|
|
||||||
metadata = {
|
|
||||||
"file": str(filepath),
|
|
||||||
"name": enum_name,
|
|
||||||
"type": "enum",
|
|
||||||
"line": start_line + 1,
|
|
||||||
"language": "rust",
|
|
||||||
"variants": variants
|
|
||||||
}
|
|
||||||
|
|
||||||
chunks.append({"text": chunk_text, "metadata": metadata})
|
|
||||||
|
|
||||||
# Parse traits
|
|
||||||
for match in trait_pattern.finditer(code):
|
|
||||||
trait_name = match.group(1)
|
|
||||||
start_pos = match.start()
|
|
||||||
start_line = code[:start_pos].count('\n')
|
|
||||||
end_line = _find_rust_brace_block_end(lines, start_line)
|
|
||||||
trait_code = "".join(lines[start_line:end_line + 1])
|
|
||||||
|
|
||||||
# Extract method signatures from trait
|
|
||||||
methods = _extract_rust_trait_methods(trait_code)
|
|
||||||
|
|
||||||
chunk_text = f"File: {filepath}\nType: trait\nName: {trait_name}\n"
|
|
||||||
if methods:
|
|
||||||
chunk_text += f"Methods: {', '.join(methods)}\n"
|
|
||||||
chunk_text += f"Code:\n{trait_code}"
|
|
||||||
|
|
||||||
metadata = {
|
|
||||||
"file": str(filepath),
|
|
||||||
"name": trait_name,
|
|
||||||
"type": "trait",
|
|
||||||
"line": start_line + 1,
|
|
||||||
"language": "rust",
|
|
||||||
"methods": methods
|
|
||||||
}
|
|
||||||
|
|
||||||
chunks.append({"text": chunk_text, "metadata": metadata})
|
|
||||||
|
|
||||||
# Parse impl blocks
|
|
||||||
for match in impl_pattern.finditer(code):
|
|
||||||
impl_target = match.group(1)
|
|
||||||
start_pos = match.start()
|
|
||||||
start_line = code[:start_pos].count('\n')
|
|
||||||
end_line = _find_rust_brace_block_end(lines, start_line)
|
|
||||||
impl_code = "".join(lines[start_line:end_line + 1])
|
|
||||||
|
|
||||||
# Extract methods from impl block
|
|
||||||
impl_methods = _extract_rust_impl_methods(impl_code)
|
|
||||||
|
|
||||||
chunk_text = f"File: {filepath}\nType: impl\nTarget: {impl_target}\n"
|
|
||||||
if impl_methods:
|
|
||||||
chunk_text += f"Implementation Methods: {', '.join(impl_methods)}\n"
|
|
||||||
chunk_text += f"Code:\n{impl_code}"
|
|
||||||
|
|
||||||
metadata = {
|
|
||||||
"file": str(filepath),
|
|
||||||
"name": f"impl_{impl_target}",
|
|
||||||
"type": "impl",
|
|
||||||
"line": start_line + 1,
|
|
||||||
"language": "rust",
|
|
||||||
"target": impl_target,
|
|
||||||
"methods": impl_methods
|
|
||||||
}
|
|
||||||
|
|
||||||
chunks.append({"text": chunk_text, "metadata": metadata})
|
|
||||||
|
|
||||||
return chunks
|
|
||||||
|
|
||||||
def _find_rust_brace_block_end(lines, start_line):
|
|
||||||
"""Find the closing brace for Rust code blocks."""
|
|
||||||
brace_count = 0
|
|
||||||
for i in range(start_line, len(lines)):
|
|
||||||
line = lines[i]
|
|
||||||
brace_count += line.count('{')
|
|
||||||
brace_count -= line.count('}')
|
|
||||||
if brace_count == 0:
|
|
||||||
return i
|
|
||||||
return len(lines) - 1
|
|
||||||
|
|
||||||
def _extract_rust_struct_fields(struct_code):
|
|
||||||
"""Extract field names from Rust struct definition."""
|
|
||||||
fields = []
|
|
||||||
# Look for field patterns: field_name: Type,
|
|
||||||
field_matches = re.findall(r'(\w+)\s*:\s*[^,\n]+', struct_code)
|
|
||||||
fields.extend(field_matches)
|
|
||||||
return fields
|
|
||||||
|
|
||||||
def _extract_rust_enum_variants(enum_code):
|
|
||||||
"""Extract variant names from Rust enum definition."""
|
|
||||||
variants = []
|
|
||||||
# Look for variant patterns: VariantName,
|
|
||||||
variant_matches = re.findall(r'(\w+)(?:\([^)]*\))?\s*,', enum_code)
|
|
||||||
variants.extend(variant_matches)
|
|
||||||
return variants
|
|
||||||
|
|
||||||
def _extract_rust_trait_methods(trait_code):
|
|
||||||
"""Extract method names from Rust trait definition."""
|
|
||||||
methods = []
|
|
||||||
# Look for method signatures in trait
|
|
||||||
method_matches = re.findall(r'fn\s+(\w+)\s*\([^)]*\)', trait_code)
|
|
||||||
methods.extend(method_matches)
|
|
||||||
return methods
|
|
||||||
|
|
||||||
def _extract_rust_impl_methods(impl_code):
|
|
||||||
"""Extract method names from Rust impl block."""
|
|
||||||
methods = []
|
|
||||||
# Look for method implementations in impl block
|
|
||||||
method_matches = re.findall(r'fn\s+(\w+)\s*\([^)]*\)', impl_code)
|
|
||||||
methods.extend(method_matches)
|
|
||||||
return methods
|
|
||||||
|
|
||||||
@lru_cache(maxsize=1000)
|
@lru_cache(maxsize=1000)
|
||||||
def parse_shell_file_cached(filepath_str: str) -> tuple:
|
def parse_shell_file_cached(filepath_str: str) -> tuple:
|
||||||
"""Cached shell script parsing. Returns tuple for hashability."""
|
"""Cached shell script parsing. Returns tuple for hashability."""
|
||||||
|
|||||||
Reference in New Issue
Block a user