diff --git a/graph/graph.py b/graph/graph.py new file mode 100644 index 0000000..65ce56c --- /dev/null +++ b/graph/graph.py @@ -0,0 +1,110 @@ +import os +import pickle +import networkx as nx +from typing import Optional + +from enhanced_toon import EnhancedToon + +class LocalGraph: + def __init__(self, root_path: str): + self.root_path = root_path + self.graph_dir = os.path.join(root_path, ".mcp_cache", "graph") + os.makedirs(self.graph_dir, exist_ok=True) + self.graph_path = os.path.join(self.graph_dir, "graph.pkl") + self.graph = nx.DiGraph() + + # --- Creation --- + def add_node(self, node_id: str, **attrs): + self.graph.add_node(node_id, **attrs) + + def add_edge(self, src: str, dst: str, edge_type: str, **attrs): + self.graph.add_edge(src, dst, type=edge_type, **attrs) + + # --- Persistence --- + def save(self): + with open(self.graph_path, "wb") as f: + pickle.dump(self.graph, f, protocol=pickle.HIGHEST_PROTOCOL) + + def load(self) -> bool: + if not os.path.exists(self.graph_path): + return False + with open(self.graph_path, "rb") as f: + self.graph = pickle.load(f) + return True + + def clear(self): + self.graph = nx.DiGraph() + if os.path.exists(self.graph_path): + os.remove(self.graph_path) + + # --- Query helpers --- + def find_nodes(self, **filters): + for node, data in self.graph.nodes(data=True): + if all(data.get(k) == v for k, v in filters.items()): + yield node, data + + def neighbors(self, node_id, edge_type: Optional[str] = None): + for n in self.graph.successors(node_id): + if not edge_type or self.graph.edges[node_id, n]["type"] == edge_type: + yield n, self.graph.nodes[n] + + def to_json(self): + out = { + "nodes": [ + {"id": n, **data} for n, data in self.graph.nodes(data=True) + ], + "edges": [ + {"src": s, "dst": d, **data} for s, d, data in self.graph.edges(data=True) + ], + } + path = os.path.join(self.graph_dir, "graph.json") + with open(path, "w") as f: + import json + json.dump(out, f, indent=2) + return path + + def to_toon(self): + """ + Export graph to compact TOON (Token-Oriented Object Notation) format. + This reduces token count by 30-60% for LLM consumption. + """ + node_list = [] + for node_id, data in self.graph.nodes(data=True): + # Build a flattened row + row = { + "id": node_id, + "type": data.get("type", ""), + "name": data.get("name", ""), + "lang": data.get("lang", ""), + "file": data.get("file", ""), + } + node_list.append(row) + + edge_list = [] + for src, dst, data in self.graph.edges(data=True): + edge_list.append({ + "src": src, + "dst": dst, + "type": data.get("type", "") + }) + + lines = [ + "# Graph Export", + f"nodes[{len(node_list)}]{{id,type,name,lang,file}}:" + ] + for n in node_list: + row = [n["id"], n["type"], n["name"], n["lang"], n["file"]] + escaped = [EnhancedToon._escape_toon_field(str(f)) for f in row] + lines.append(" " + ",".join(escaped)) + + lines.append(f"edges[{len(edge_list)}]{{src,dst,type}}:") + for e in edge_list: + row = [e["src"], e["dst"], e["type"]] + escaped = [EnhancedToon._escape_toon_field(str(f)) for f in row] + lines.append(" " + ",".join(escaped)) + + toon_text = "\n".join(lines) + path = os.path.join(self.graph_dir, "graph.toon") + with open(path, "w", encoding="utf-8") as f: + f.write(toon_text) + return path diff --git a/mcp_codebase.py b/mcp_codebase.py index 311a869..b0da96a 100644 --- a/mcp_codebase.py +++ b/mcp_codebase.py @@ -26,6 +26,8 @@ import sqlparse import sqlglot from enhanced_toon import EnhancedToon +from graph.graph import LocalGraph + os.environ["ANONYMIZED_TELEMETRY"] = "False" @@ -66,16 +68,48 @@ CONTEXTUAL_WEIGHTS = { "sql_index": 1.1, # INDEX definitions # Python weights - "python_class": 1.2, # Class definitions + "python_class": 1.25, # Class definitions (enhanced) "python_function": 1.15, # Function definitions + "python_method": 1.2, # Method definitions (enhanced) + "python_async": 1.1, # Async functions # Go weights - "go_type": 1.2, # Type definitions + "go_type": 1.25, # Type definitions (struct, interface) "go_function": 1.15, # Function definitions + "go_method": 1.2, # Method definitions (enhanced) + "go_struct": 1.3, # Struct definitions (enhanced) # Java weights - "java_class": 1.2, # Class definitions - "java_method": 1.15, # Method definitions + "java_class": 1.3, # Class definitions (enhanced) + "java_interface": 1.25, # Interface definitions (enhanced) + "java_method": 1.2, # Method definitions (enhanced) + "java_constructor": 1.15, # Constructor definitions (enhanced) + "java_enum": 1.2, # Enum definitions (enhanced) + "java_field": 1.1, # Field definitions (enhanced) + + # Rust weights (new) + "rust_struct": 1.3, # Struct definitions + "rust_enum": 1.25, # Enum definitions + "rust_trait": 1.35, # Trait definitions (very important in Rust) + "rust_impl": 1.2, # Implementation blocks + "rust_function": 1.15, # Function definitions + "rust_method": 1.2, # Method definitions + "rust_async": 1.1, # Async functions + "rust_unsafe": 1.15, # Unsafe blocks/functions + + # TypeScript/JavaScript weights (for Svelte scripts) + "typescript_interface": 1.25, # Interface definitions + "typescript_class": 1.2, # Class definitions + "typescript_function": 1.15, # Function definitions + "typescript_type": 1.2, # Type definitions + + # Svelte weights (new) + "svelte_component": 1.4, # Component definitions (very important) + "svelte_script": 1.1, # Script blocks + "svelte_style": 1.05, # Style blocks + "svelte_markup": 1.15, # Markup/template + "svelte_prop": 1.25, # Component props (enhanced) + "svelte_reactive": 1.2, # Reactive declarations # Default weight for unspecified types "default": 1.0 @@ -133,6 +167,7 @@ def calculate_contextual_weight(metadata: Dict[str, Any]) -> float: """ language = metadata.get("language", "").lower() chunk_type = metadata.get("type", "").lower() + block_type = metadata.get("block_type", "").lower() # SQL-specific weights if language == "sql": @@ -145,30 +180,97 @@ def calculate_contextual_weight(metadata: Dict[str, Any]) -> float: elif chunk_type in ("index", "create index"): return CONTEXTUAL_WEIGHTS.get("sql_index", 1.0) - # Python-specific weights + # Python-specific weights (enhanced) elif language == "python": if chunk_type == "classdef": return CONTEXTUAL_WEIGHTS.get("python_class", 1.0) elif chunk_type in ("functiondef", "asyncfunctiondef"): - return CONTEXTUAL_WEIGHTS.get("python_function", 1.0) + if metadata.get("is_method"): + return CONTEXTUAL_WEIGHTS.get("python_method", 1.0) + elif chunk_type == "asyncfunctiondef" or metadata.get("is_async"): + return CONTEXTUAL_WEIGHTS.get("python_async", 1.0) + else: + return CONTEXTUAL_WEIGHTS.get("python_function", 1.0) - # Go-specific weights + # Go-specific weights (enhanced) elif language == "go": - if chunk_type in ("type", "struct", "interface"): + if chunk_type in ("struct", "interface"): + return CONTEXTUAL_WEIGHTS.get("go_struct", 1.0) + elif chunk_type == "type": return CONTEXTUAL_WEIGHTS.get("go_type", 1.0) - elif chunk_type in ("func", "method"): + elif chunk_type == "method": + return CONTEXTUAL_WEIGHTS.get("go_method", 1.0) + elif chunk_type == "func": return CONTEXTUAL_WEIGHTS.get("go_function", 1.0) - # Java-specific weights + # Java-specific weights (enhanced) elif language == "java": if chunk_type == "class": return CONTEXTUAL_WEIGHTS.get("java_class", 1.0) + elif chunk_type == "interface": + return CONTEXTUAL_WEIGHTS.get("java_interface", 1.0) elif chunk_type == "method": return CONTEXTUAL_WEIGHTS.get("java_method", 1.0) + elif chunk_type == "constructor": + return CONTEXTUAL_WEIGHTS.get("java_constructor", 1.0) + elif chunk_type == "enum": + return CONTEXTUAL_WEIGHTS.get("java_enum", 1.0) + elif chunk_type == "field": + return CONTEXTUAL_WEIGHTS.get("java_field", 1.0) + + # Rust-specific weights (new) + elif language == "rust": + if chunk_type == "struct": + return CONTEXTUAL_WEIGHTS.get("rust_struct", 1.0) + elif chunk_type == "enum": + return CONTEXTUAL_WEIGHTS.get("rust_enum", 1.0) + elif chunk_type == "trait": + return CONTEXTUAL_WEIGHTS.get("rust_trait", 1.0) + elif chunk_type == "impl": + return CONTEXTUAL_WEIGHTS.get("rust_impl", 1.0) + elif chunk_type == "function": + if metadata.get("is_async"): + return CONTEXTUAL_WEIGHTS.get("rust_async", 1.0) + elif metadata.get("is_unsafe"): + return CONTEXTUAL_WEIGHTS.get("rust_unsafe", 1.0) + elif metadata.get("is_method", False): + return CONTEXTUAL_WEIGHTS.get("rust_method", 1.0) + else: + return CONTEXTUAL_WEIGHTS.get("rust_function", 1.0) + + # TypeScript/JavaScript weights (for Svelte scripts) + elif language == "typescript": + if chunk_type == "interface": + return CONTEXTUAL_WEIGHTS.get("typescript_interface", 1.0) + elif chunk_type == "class": + return CONTEXTUAL_WEIGHTS.get("typescript_class", 1.0) + elif chunk_type == "function": + return CONTEXTUAL_WEIGHTS.get("typescript_function", 1.0) + elif chunk_type == "type": + return CONTEXTUAL_WEIGHTS.get("typescript_type", 1.0) + + # Svelte-specific weights (new) + elif language == "svelte": + if chunk_type == "component": + return CONTEXTUAL_WEIGHTS.get("svelte_component", 1.0) + elif chunk_type == "prop": + return CONTEXTUAL_WEIGHTS.get("svelte_prop", 1.0) + elif chunk_type == "reactive": + return CONTEXTUAL_WEIGHTS.get("svelte_reactive", 1.0) + elif block_type == "markup": + return CONTEXTUAL_WEIGHTS.get("svelte_markup", 1.0) + elif block_type == "script": + return CONTEXTUAL_WEIGHTS.get("svelte_script", 1.0) + elif block_type == "style": + return CONTEXTUAL_WEIGHTS.get("svelte_style", 1.0) + + # CSS weights (for Svelte styles) + elif language == "css": + if block_type == "style": + return CONTEXTUAL_WEIGHTS.get("svelte_style", 1.0) return CONTEXTUAL_WEIGHTS.get("default", 1.0) - def apply_contextual_weights_to_embeddings(embeddings_list: List[List[float]], metadata_list: List[Dict[str, Any]]) -> List[List[float]]: """ @@ -220,33 +322,209 @@ def load_ragignore(base_path: Path) -> pathspec.PathSpec: # ----------------------------- @lru_cache(maxsize=1000) def parse_python_file_cached(filepath_str: str, file_hash: str) -> tuple: - """Cached Python AST parsing. Returns tuple for hashability.""" + """Cached Python AST parsing with enhanced metadata for graph relationships.""" filepath = Path(filepath_str) chunks = [] try: code = filepath.read_text(encoding="utf-8") tree = ast.parse(code) lines = code.splitlines(keepends=True) + + # Track imports for module relationships + imports = [] + for node in ast.walk(tree): + # Extract imports for module dependencies + if isinstance(node, (ast.Import, ast.ImportFrom)): + import_info = _extract_import_info(node) + if import_info: + imports.append(import_info) + if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef)): docstring = ast.get_docstring(node) or "" start_line = node.lineno - 1 end_line = getattr(node, "end_lineno", start_line + 1) chunk_code = "".join(lines[start_line:end_line]) + + # Extract enhanced metadata + metadata = { + "file": str(filepath), + "name": getattr(node, "name", ""), + "type": type(node).__name__, + "line": node.lineno, + "language": "python", + } + + # Add specific metadata based on node type + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + metadata.update(_extract_function_metadata(node, code)) + elif isinstance(node, ast.ClassDef): + metadata.update(_extract_class_metadata(node, code)) + + # Include imports in the context + if imports: + metadata["imports"] = imports + + chunk_text = f"File: {filepath}\nType: {type(node).__name__}\nName: {getattr(node,'name', '')}\nDocstring: {docstring}\n" + + # Add enhanced information to text + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = _format_arguments(node.args) + returns = _extract_return_annotation(node) + chunk_text += f"Arguments: {args}\n" + if returns: + chunk_text += f"Returns: {returns}\n" + elif isinstance(node, ast.ClassDef): + bases = [ast.unparse(base) for base in node.bases] if hasattr(ast, 'unparse') else [ast.dump(base) for base in node.bases] + if bases: + chunk_text += f"Base Classes: {', '.join(bases)}\n" + + chunk_text += f"\nCode:\n{chunk_code}" + chunks.append({ - "text": f"File: {filepath}\nType: {type(node).__name__}\nName: {getattr(node,'name', '')}\nDocstring: {docstring}\n\nCode:\n{chunk_code}", - "metadata": { - "file": str(filepath), - "name": getattr(node, "name", ""), - "type": type(node).__name__, - "line": node.lineno, - "language": "python", - } + "text": chunk_text, + "metadata": metadata }) + except Exception as e: logger.warning(f"parse_python_file: failed {filepath}: {e}") return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) +def _extract_import_info(node): + """Extract import information for module dependencies.""" + if isinstance(node, ast.Import): + return { + "type": "import", + "modules": [alias.name for alias in node.names], + "level": 0 + } + elif isinstance(node, ast.ImportFrom): + return { + "type": "import_from", + "module": node.module, + "names": [alias.name for alias in node.names], + "level": node.level + } + return None + +def _extract_function_metadata(node, code): + """Extract enhanced metadata for functions/methods.""" + metadata = {} + + # Extract arguments + args = node.args + arg_names = [arg.arg for arg in args.args] + if args.vararg: + arg_names.append(f"*{args.vararg.arg}") + if args.kwarg: + arg_names.append(f"**{args.kwarg.arg}") + + metadata["parameters"] = arg_names + + # Extract decorators + decorators = [] + for decorator in node.decorator_list: + if isinstance(decorator, ast.Name): + decorators.append(decorator.id) + elif isinstance(decorator, ast.Attribute): + decorators.append(ast.unparse(decorator) if hasattr(ast, 'unparse') else ast.dump(decorator)) + elif isinstance(decorator, ast.Call): + decorators.append(ast.unparse(decorator.func) if hasattr(ast, 'unparse') else ast.dump(decorator.func)) + + if decorators: + metadata["decorators"] = decorators + + # Try to extract return annotation + return_annotation = _extract_return_annotation(node) + if return_annotation: + metadata["return_type"] = return_annotation + + # Detect if it's a method (has 'self' or 'cls' as first argument) + if arg_names and arg_names[0] in ('self', 'cls'): + metadata["is_method"] = True + # Try to find the containing class + metadata["method_type"] = "classmethod" if arg_names[0] == 'cls' else "instancemethod" + else: + metadata["is_method"] = False + + return metadata + +def _extract_class_metadata(node, code): + """Extract enhanced metadata for classes.""" + metadata = {} + + # Extract base classes + bases = [] + for base in node.bases: + if isinstance(base, ast.Name): + bases.append(base.id) + elif isinstance(base, ast.Attribute): + bases.append(ast.unparse(base) if hasattr(ast, 'unparse') else ast.dump(base)) + + if bases: + metadata["base_classes"] = bases + + # Extract decorators + decorators = [] + for decorator in node.decorator_list: + if isinstance(decorator, ast.Name): + decorators.append(decorator.id) + + if decorators: + metadata["decorators"] = decorators + + # Try to detect class type from common patterns + class_code = ast.unparse(node) if hasattr(ast, 'unparse') else ast.dump(node) + if "metaclass" in class_code: + metadata["has_metaclass"] = True + + # Look for common base class patterns + for base in bases: + if base in ("Exception", "BaseException"): + metadata["class_type"] = "exception" + break + elif base in ("Enum", "IntEnum", "StrEnum"): + metadata["class_type"] = "enum" + break + elif base in ("Model", "BaseModel"): + metadata["class_type"] = "model" + break + + return metadata + +def _format_arguments(args): + """Format function arguments in a readable way.""" + parts = [] + + # Positional arguments + for arg in args.args: + parts.append(arg.arg) + + # *args + if args.vararg: + parts.append(f"*{args.vararg.arg}") + + # Keyword-only arguments + for arg in args.kwonlyargs: + parts.append(arg.arg) + + # **kwargs + if args.kwarg: + parts.append(f"**{args.kwarg.arg}") + + return ", ".join(parts) + +def _extract_return_annotation(node): + """Extract return type annotation if available.""" + if hasattr(node, 'returns') and node.returns: + if isinstance(node.returns, ast.Name): + return node.returns.id + elif isinstance(node.returns, ast.Attribute): + return ast.unparse(node.returns) if hasattr(ast, 'unparse') else ast.dump(node.returns) + elif isinstance(node.returns, ast.Subscript): + return ast.unparse(node.returns) if hasattr(ast, 'unparse') else ast.dump(node.returns) + return None + def parse_python_file(filepath: Path) -> List[Dict]: """Parse Python file with caching.""" file_hash = get_file_hash(filepath) @@ -412,118 +690,7 @@ def parse_sql_file_cached(filepath_str: str, file_hash: str) -> tuple: chunks.append({"text": chunk_text, "metadata": meta}) return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) - """Cached SQL parsing. Returns tuple for hashability.""" - filepath = Path(filepath_str) - try: - sql_code = filepath.read_text(encoding="utf-8") - except Exception as e: - logger.warning(f"parse_sql_file: cannot read {filepath}: {e}") - return tuple() - - chunks: List[Dict[str, Any]] = [] - statements = [s for s in sqlparse.split(sql_code) if s and s.strip()] - lines = sql_code.splitlines() - search_pos = 0 - - for stmt in statements: - # find position of this statement in the original SQL (using running search_pos) - start_idx = sql_code.find(stmt, search_pos) - if start_idx == -1: - # fallback: try from beginning - start_idx = sql_code.find(stmt) - if start_idx == -1: - # cannot locate, approximate line number as last line - start_line = len(lines) - else: - start_line = sql_code[:start_idx].count("\n") + 1 - search_pos = start_idx + len(stmt) - - # include leading comments above this statement - leading_comments = include_leading_comments(lines, start_line) - - # Trim and normalize statement - stmt_clean = stmt.strip() - stmt_text_for_embed = (leading_comments + "\n\n" + stmt_clean) if leading_comments else stmt_clean - - # Try to parse with sqlglot to extract tables / functions / ctes metadata - meta: Dict[str, Any] = { - "file": str(filepath), - "name": None, - "type": None, - "line": start_line, - "language": "sql" - } - try: - parsed = sqlglot.parse_one(stmt_clean, read="postgres") - # stmt type - stmt_type = getattr(parsed, "key", None) or parsed.token_type if hasattr(parsed, "token_type") else None - meta["type"] = str(stmt_type).lower() if stmt_type else "statement" - - # table names (if any) - tables = [t.this for t in parsed.find_all(sqlglot.exp.Table)] - meta["tables"] = [t for t in tables if isinstance(t, str)] - # ctes - ctes = [] - for cte in parsed.find_all(sqlglot.exp.CTE): - try: - alias = cte.alias_or_name - if alias: - ctes.append(alias) - except Exception: - pass - meta["ctes"] = ctes - - # functions called in the statement (names) - funcs = [] - for f in parsed.find_all(sqlglot.exp.Func): - try: - name = f.name - if name: - funcs.append(name) - except Exception: - pass - meta["functions"] = funcs - - # choose a sensible name for the metadata - if meta.get("tables"): - meta["name"] = meta["tables"][0] - elif ctes: - meta["name"] = ctes[0] - elif funcs: - meta["name"] = funcs[0] - - chunk_text = f"File: {filepath}\nType: {meta.get('type')}\nName: {meta.get('name')}\n\nComments:\n{leading_comments}\n\nSQL:\n{stmt_clean}" if leading_comments else f"File: {filepath}\nType: {meta.get('type')}\nName: {meta.get('name')}\n\nSQL:\n{stmt_clean}" - - chunks.append({ - "text": chunk_text, - "metadata": meta - }) - - continue - except Exception: - # sqlglot failed on this statement — fall back to heuristics (still keep comments) - pass - - # Fallback: simple heuristics if sqlglot fails - # Determine an approximate type by the first token - try: - parsed_tok = sqlparse.parse(stmt_clean)[0] - first_token = parsed_tok.token_first(skip_cm=True) - stmt_type = first_token.value.upper() if first_token else "UNKNOWN" - except Exception: - stmt_type = "UNKNOWN" - - meta["type"] = stmt_type.lower() - chunk_text = f"File: {filepath}\nType: {meta['type']}\nName: {meta.get('name')}\n\nComments:\n{leading_comments}\n\nSQL:\n{stmt_clean}" if leading_comments else f"File: {filepath}\nType: {meta['type']}\nName: {meta.get('name')}\n\nSQL:\n{stmt_clean}" - - chunks.append({ - "text": chunk_text, - "metadata": meta - }) - pass - - return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) - + def parse_sql_file(filepath: Path) -> List[Dict[str, Any]]: """Parse SQL file with caching.""" file_hash = get_file_hash(filepath) @@ -603,7 +770,7 @@ def include_leading_comments(lines: List[str], stmt_start_line: int, max_context @lru_cache(maxsize=1000) def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: - """Cached Go AST parsing. Returns tuple for hashability.""" + """Cached Go AST parsing with enhanced metadata for graph relationships.""" filepath = Path(filepath_str) chunks = [] helper = Path("./tools/parse_go_ast") @@ -628,6 +795,8 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: typ = d.get("type", "") doc_comment = d.get("doc_comment", "") receiver = d.get("receiver", "") + fields = d.get("fields", []) + methods = d.get("methods", []) # Skip package declarations if typ == "package": @@ -643,18 +812,36 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: chunk_text = f"File: {filepath}\nType: {display_type}\nName: {name}\n" if doc_comment: chunk_text += f"Doc:\n{doc_comment}\n\n" + if fields: + chunk_text += f"Fields: {', '.join(fields)}\n" + if methods: + chunk_text += f"Methods: {', '.join(methods)}\n" chunk_text += f"Code:\n{chunk_code}" + # Build comprehensive metadata + metadata = { + "file": str(filepath), + "name": name, + "type": typ, + "receiver": receiver, + "line": start + 1, + "language": "go", + } + + # Enhanced metadata for graph relationships + if fields: + metadata["fields"] = fields + if methods: + metadata["interface_methods"] = methods + if receiver: + metadata["receiver_type"] = receiver + # Extract receiver type without pointer for type matching + receiver_base = receiver.replace('*', '') + metadata["receiver_base_type"] = receiver_base + chunks.append({ "text": chunk_text, - "metadata": { - "file": str(filepath), - "name": name, - "type": typ, - "receiver": receiver, - "line": start + 1, - "language": "go", - } + "metadata": metadata }) if chunks: # If helper worked, return its results @@ -667,7 +854,7 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: except Exception as e: logger.warning(f"parse_go_file helper failed for {filepath}: {e}; falling back to regex") - # Fallback: Use regex-based parsing (same as before) + # Fallback: Use regex-based parsing try: content = filepath.read_text(encoding="utf-8") lines = content.splitlines(keepends=True) @@ -686,23 +873,42 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: func_code = "".join(lines[start_line:end_line + 1]) comments = extract_go_comments(lines, start_line) + # Try to extract receiver type from function signature + receiver_type = "" + func_sig = content[start_pos:match.end()] + receiver_match = re.search(r'func\s*\(([^)]+)\)', func_sig) + if receiver_match: + receiver_part = receiver_match.group(1) + # Extract type from receiver (e.g., "u *User" -> "*User") + receiver_type_match = re.search(r'\*?(\w+)', receiver_part.split()[-1] if ' ' in receiver_part else receiver_part) + if receiver_type_match: + receiver_type = receiver_type_match.group(0) + chunk_text = f"File: {filepath}\nType: function\nName: {func_name}\n" if comments: chunk_text += f"Comments:\n{comments}\n\n" chunk_text += f"Code:\n{func_code}" + metadata = { + "file": str(filepath), + "name": func_name, + "type": "function", + "line": start_line + 1, + "language": "go", + } + + # Receiver type for methods + if receiver_type: + metadata["receiver_type"] = receiver_type + metadata["receiver_base_type"] = receiver_type.replace('*', '') + metadata["type"] = "method" + chunks.append({ "text": chunk_text, - "metadata": { - "file": str(filepath), - "name": func_name, - "type": "function", - "line": start_line + 1, - "language": "go", - } + "metadata": metadata }) - # Structs + # Structs with field extraction struct_pattern = re.compile(r'^type\s+(\w+)\s+struct\s*\{', re.MULTILINE) for match in struct_pattern.finditer(content): struct_name = match.group(1) @@ -711,24 +917,42 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: end_line = find_brace_block_end_go(lines, start_line) struct_code = "".join(lines[start_line:end_line + 1]) comments = extract_go_comments(lines, start_line) + + # Extract field names from struct + field_names = [] + struct_body_match = re.search(r'struct\s*\{([^}]+)\}', struct_code, re.DOTALL) + if struct_body_match: + field_lines = struct_body_match.group(1).split('\n') + for line in field_lines: + field_match = re.match(r'\s*(\w+)\s+', line.strip()) + if field_match: + field_names.append(field_match.group(1)) chunk_text = f"File: {filepath}\nType: struct\nName: {struct_name}\n" if comments: chunk_text += f"Comments:\n{comments}\n\n" + if field_names: + chunk_text += f"Fields: {', '.join(field_names)}\n" chunk_text += f"Code:\n{struct_code}" + metadata = { + "file": str(filepath), + "name": struct_name, + "type": "struct", + "line": start_line + 1, + "language": "go", + } + + # Struct fields for graph relationships + if field_names: + metadata["fields"] = field_names + chunks.append({ "text": chunk_text, - "metadata": { - "file": str(filepath), - "name": struct_name, - "type": "struct", - "line": start_line + 1, - "language": "go", - } + "metadata": metadata }) - # Interfaces + # Interfaces with method extraction interface_pattern = re.compile(r'^type\s+(\w+)\s+interface\s*\{', re.MULTILINE) for match in interface_pattern.finditer(content): interface_name = match.group(1) @@ -737,21 +961,39 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: end_line = find_brace_block_end_go(lines, start_line) interface_code = "".join(lines[start_line:end_line + 1]) comments = extract_go_comments(lines, start_line) + + # Extract method signatures from interface + method_names = [] + interface_body_match = re.search(r'interface\s*\{([^}]+)\}', interface_code, re.DOTALL) + if interface_body_match: + method_lines = interface_body_match.group(1).split('\n') + for line in method_lines: + method_match = re.match(r'\s*(\w+)\s*\([^)]*\)', line.strip()) + if method_match: + method_names.append(method_match.group(1)) chunk_text = f"File: {filepath}\nType: interface\nName: {interface_name}\n" if comments: chunk_text += f"Comments:\n{comments}\n\n" + if method_names: + chunk_text += f"Methods: {', '.join(method_names)}\n" chunk_text += f"Code:\n{interface_code}" + metadata = { + "file": str(filepath), + "name": interface_name, + "type": "interface", + "line": start_line + 1, + "language": "go", + } + + # Interface methods for graph relationships + if method_names: + metadata["interface_methods"] = method_names + chunks.append({ "text": chunk_text, - "metadata": { - "file": str(filepath), - "name": interface_name, - "type": "interface", - "line": start_line + 1, - "language": "go", - } + "metadata": metadata }) except Exception as e: @@ -843,7 +1085,7 @@ def parse_go_file(filepath: Path) -> List[Dict]: # Java parsing (javalang + manual body extraction) with caching # ----------------------------- @lru_cache(maxsize=1000) -def parse_java_file_cached(filepath_str: str, file_hash: str) -> tuple: +def parse_java_file_cached(filepath_str: str) -> tuple: """Cached Java parsing. Returns tuple for hashability.""" filepath = Path(filepath_str) chunks = [] @@ -870,14 +1112,24 @@ def parse_java_file_cached(filepath_str: str, file_hash: str) -> tuple: end_line = find_brace_block_end(start_line) chunk_text = "".join(lines[start_line:end_line + 1]) doc = get_context_with_comments(filepath, start_line + 1) + extends_class = None + implements_interfaces = [] + + if node.extends: + extends_class = node.extends.name + if hasattr(node, 'implements') and node.implements: + implements_interfaces = [impl.name for impl in node.implements] + chunks.append({ - "text": f"File: {filepath}\nType: Class\nName: {node.name}\nDoc:\n{doc}\nCode:\n{chunk_text}", + "text": f"File: {filepath}\nType: Class\nName: {node.name}\nExtends: {extends_class or 'None'}\nImplements: {', '.join(implements_interfaces) or 'None'}\nDoc:\n{doc}\nCode:\n{chunk_text}", "metadata": { "file": str(filepath), "name": node.name, "type": "class", "line": start_line + 1, - "language": "java" + "language": "java", + "extends": extends_class, + "implements": implements_interfaces } }) @@ -894,15 +1146,24 @@ def parse_java_file_cached(filepath_str: str, file_hash: str) -> tuple: if isinstance(p, javalang.tree.ClassDeclaration): class_name = p.name break + return_type = node.return_type.name if node.return_type else "void" + parameters = [] + if node.parameters: + for param in node.parameters: + param_type = param.type.name if param.type else "unknown" + parameters.append(f"{param_type} {param.name}") + chunks.append({ - "text": f"File: {filepath}\nType: Method\nClass: {class_name or 'unknown'}\nName: {node.name}\nDoc:\n{doc}\nCode:\n{chunk_text}", + "text": f"File: {filepath}\nType: Method\nClass: {class_name or 'unknown'}\nName: {node.name}\nReturn: {return_type}\nParameters: {', '.join(parameters) or 'None'}\nDoc:\n{doc}\nCode:\n{chunk_text}", "metadata": { "file": str(filepath), "name": node.name, "type": "method", "class": class_name, "line": start_line + 1, - "language": "java" + "language": "java", + "return_type": return_type, + "parameters": parameters } }) @@ -915,15 +1176,17 @@ def parse_java_file_cached(filepath_str: str, file_hash: str) -> tuple: end_line = start_line chunk_text = lines[start_line] doc = get_context_with_comments(filepath, start_line + 1) + field_type = node.type.name if node.type else "unknown" for declarator in node.declarators: chunks.append({ - "text": f"File: {filepath}\nType: Field\nName: {declarator.name}\nDoc:\n{doc}\nCode:\n{chunk_text}", + "text": f"File: {filepath}\nType: Field\nName: {declarator.name}\nField Type: {field_type}\nDoc:\n{doc}\nCode:\n{chunk_text}", "metadata": { "file": str(filepath), "name": declarator.name, "type": "field", "line": start_line + 1, - "language": "java" + "language": "java", + "field_type": field_type } }) @@ -966,6 +1229,38 @@ def parse_java_file_cached(filepath_str: str, file_hash: str) -> tuple: "language": "java" } }) + + elif isinstance(node, javalang.tree.InterfaceDeclaration): + start_line = node.position.line - 1 if node.position else 0 + end_line = find_brace_block_end(start_line) + chunk_text = "".join(lines[start_line:end_line + 1]) + doc = get_context_with_comments(filepath, start_line + 1) + chunks.append({ + "text": f"File: {filepath}\nType: Interface\nName: {node.name}\nDoc:\n{doc}\nCode:\n{chunk_text}", + "metadata": { + "file": str(filepath), + "name": node.name, + "type": "interface", + "line": start_line + 1, + "language": "java" + } + }) + + elif isinstance(node, javalang.tree.AnnotationDeclaration): + start_line = node.position.line - 1 if node.position else 0 + end_line = find_brace_block_end(start_line) + chunk_text = "".join(lines[start_line:end_line + 1]) + doc = get_context_with_comments(filepath, start_line + 1) + chunks.append({ + "text": f"File: {filepath}\nType: Annotation\nName: {node.name}\nDoc:\n{doc}\nCode:\n{chunk_text}", + "metadata": { + "file": str(filepath), + "name": node.name, + "type": "annotation", + "line": start_line + 1, + "language": "java" + } + }) except javalang.parser.JavaSyntaxError as e: logger.warning(f"Failed to parse Java file {filepath}: {e}") @@ -975,8 +1270,7 @@ def parse_java_file_cached(filepath_str: str, file_hash: str) -> tuple: def parse_java_file(filepath: Path) -> List[Dict]: """Parse Java source file with caching.""" - file_hash = get_file_hash(filepath) - cached_result = parse_java_file_cached(str(filepath), file_hash) + cached_result = parse_java_file_cached(str(filepath)) return [{"text": text, "metadata": dict(meta)} for text, meta in cached_result] # ----------------------------- @@ -984,17 +1278,34 @@ def parse_java_file(filepath: Path) -> List[Dict]: # ----------------------------- @lru_cache(maxsize=500) def parse_svelte_file_cached(filepath_str: str, file_hash: str) -> tuple: - """Cached Svelte parsing. Returns tuple for hashability.""" + """Cached Svelte parsing with enhanced multi-language graph relationships.""" filepath = Path(filepath_str) chunks = [] try: text = filepath.read_text(encoding="utf-8") lines = text.splitlines(keepends=True) + + # Track component props and context for graph relationships + component_props = [] + component_context = [] + + # Extract component name from filename for graph relationships + component_name = filepath.stem + if component_name[0].islower(): + component_name = component_name[0].upper() + component_name[1:] # PascalCase + script_matches = list(re.finditer(r"]*)?>(.*?)", text, flags=re.DOTALL)) idx = 0 + for m in script_matches: script_content = m.group(1) script_start_line = text[:m.start(1)].count("\n") + 1 + script_attrs = m.group(0).split('>')[0] # Get ", "", text, flags=re.DOTALL) markup = re.sub(r"]*>.*?", "", markup, flags=re.DOTALL).strip() + if markup: + # Extract component usage and bindings from markup + used_components = re.findall(r'<([A-Z][a-zA-Z]*)', markup) + prop_bindings = re.findall(r'(\w+)={([^}]+)}', markup) + event_handlers = re.findall(r'on:(\w+)=', markup) + + chunk_text = f"File: {filepath}\nType: markup\nComponent: {component_name}\n" + if used_components: + chunk_text += f"Uses Components: {', '.join(set(used_components))}\n" + if prop_bindings: + chunk_text += f"Prop Bindings: {', '.join([f'{prop}={val}' for prop, val in prop_bindings[:5]])}\n" + if event_handlers: + chunk_text += f"Event Handlers: {', '.join(set(event_handlers))}\n" + chunk_text += f"Markup:\n{markup}" + chunks.append({ - "text": f"File: {filepath}\nType: markup\n{markup}", + "text": chunk_text, "metadata": { "file": str(filepath), "name": "markup", "type": "markup", "line": 1, - "language": "svelte", - "block_type": "markup" + "language": "svelte", + "block_type": "markup", + "component_name": component_name, + "used_components": list(set(used_components)), + "prop_bindings": [prop for prop, _ in prop_bindings], + "event_handlers": list(set(event_handlers)), + "component_props": component_props, # Props discovered in scripts + "reactive_vars": component_context # Reactive context discovered } }) + except Exception as e: logger.warning(f"parse_svelte_file failed {filepath}: {e}") return tuple() return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) +def _make_script_chunk(filepath, script_content, start_line, block_type, component_name=None): + """Enhanced script chunk creation with basic analysis.""" + # Basic heuristic analysis for when TypeScript parser fails + exports = re.findall(r'export\s+(?:let|const|function|class)\s+(\w+)', script_content) + functions = re.findall(r'(?:export\s+)?function\s+(\w+)', script_content) + imports = re.findall(r'import.*from\s+[\'"]([^\'"]+)[\'"]', script_content) + + chunk_text = f"File: {filepath}\nType: {block_type}\n" + if component_name: + chunk_text += f"Component: {component_name}\n" + if exports: + chunk_text += f"Exports: {', '.join(exports)}\n" + if functions: + chunk_text += f"Functions: {', '.join(functions)}\n" + chunk_text += f"Code:\n{script_content}" + + metadata = { + "file": str(filepath), + "name": f"{block_type}_chunk", + "type": block_type, + "line": start_line, + "language": "typescript" if block_type == "script" else "css", + "block_type": block_type + } + + if component_name: + metadata["component_name"] = component_name + if exports: + metadata["exports"] = exports + if imports: + metadata["imports"] = imports + + return {"text": chunk_text, "metadata": metadata} + def parse_svelte_file(filepath: Path) -> List[Dict]: """Parse Svelte file with caching.""" file_hash = get_file_hash(filepath) @@ -1078,20 +1492,6 @@ def parse_svelte_file(filepath: Path) -> List[Dict]: return chunk_text_file(filepath, "svelte") return [{"text": text, "metadata": dict(meta)} for text, meta in cached_result] - -def _make_script_chunk(filepath: Path, content: str, start_line: int, block_type: str = "script") -> Dict: - return { - "text": f"File: {filepath}\nType: {block_type}\n{content}", - "metadata": { - "file": str(filepath), - "name": f"{block_type}_chunk", - "type": block_type, - "line": start_line, - "language": "typescript" if block_type == "script" else "css", - "block_type": block_type - } - } - # ----------------------------- # Heuristic chunking for other languages # ----------------------------- @@ -1124,8 +1524,336 @@ def chunk_text_file(filepath: Path, language: str) -> List[Dict]: }) return chunks +def parse_rust_file(filepath: Path) -> List[Dict]: + """Parse Rust file with caching.""" + file_hash = get_file_hash(filepath) + cached_result = parse_rust_file_cached(str(filepath), file_hash) + return [{"text": text, "metadata": dict(meta)} for text, meta in cached_result] + @lru_cache(maxsize=1000) -def parse_shell_file_cached(filepath_str: str, file_hash: str) -> tuple: +def parse_rust_file_cached(filepath_str: str, file_hash: str) -> tuple: + """Cached Rust parsing with enhanced metadata for graph relationships.""" + filepath = Path(filepath_str) + chunks = [] + try: + code = filepath.read_text(encoding="utf-8") + lines = code.splitlines(keepends=True) + + # Try using rust-analyzer AST if available + rust_helper = Path("./tools/parse_rust_ast") + if rust_helper.exists(): + try: + proc = subprocess.run( + [str(rust_helper), str(filepath)], + capture_output=True, + text=True, + check=True, + timeout=20 + ) + decls = json.loads(proc.stdout) + + for d in decls: + start_line = max(0, d.get("start_line", 1) - 1) + end_line = d.get("end_line", start_line + 1) + chunk_code = "".join(lines[start_line:end_line]) + + # Build comprehensive metadata + metadata = { + "file": str(filepath), + "name": d.get("name", ""), + "type": d.get("type", ""), + "line": start_line + 1, + "language": "rust", + } + + # Add Rust-specific metadata + if d.get("visibility"): + metadata["visibility"] = d.get("visibility") + if d.get("is_async"): + metadata["is_async"] = True + if d.get("is_unsafe"): + metadata["is_unsafe"] = True + if d.get("generics"): + metadata["generics"] = d.get("generics") + if d.get("traits"): + metadata["implements_traits"] = d.get("traits") + if d.get("fields"): + metadata["fields"] = d.get("fields") + if d.get("methods"): + metadata["methods"] = d.get("methods") + if d.get("return_type"): + metadata["return_type"] = d.get("return_type") + if d.get("parameters"): + metadata["parameters"] = d.get("parameters") + + chunk_text = f"File: {filepath}\nType: {d.get('type')}\nName: {d.get('name')}\n" + + # Add Rust-specific details to text + if d.get("visibility"): + chunk_text += f"Visibility: {d.get('visibility')}\n" + if d.get("is_async"): + chunk_text += "Async: yes\n" + if d.get("is_unsafe"): + chunk_text += "Unsafe: yes\n" + if d.get("generics"): + chunk_text += f"Generics: {d.get('generics')}\n" + if d.get("traits"): + chunk_text += f"Implements: {', '.join(d.get('traits', []))}\n" + if d.get("fields"): + chunk_text += f"Fields: {', '.join(d.get('fields', []))}\n" + + chunk_text += f"Code:\n{chunk_code}" + + chunks.append({ + "text": chunk_text, + "metadata": metadata + }) + + if chunks: + return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) + + except Exception as e: + logger.warning(f"Rust AST helper failed for {filepath}: {e}; falling back to regex") + + # Fallback: Regex-based Rust parsing + chunks.extend(_parse_rust_with_regex(filepath, code, lines)) + + except Exception as e: + logger.warning(f"parse_rust_file failed {filepath}: {e}") + + return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) + +def _parse_rust_with_regex(filepath: Path, code: str, lines: list) -> list: + """Fallback regex-based Rust parser.""" + chunks = [] + + # Pattern for Rust functions (including async, unsafe, methods) + func_pattern = re.compile( + r'^(?:pub\s+)?(?:async\s+)?(?:unsafe\s+)?fn\s+(\w+)\s*[<(]', + re.MULTILINE + ) + + # Pattern for structs + struct_pattern = re.compile( + r'^(?:pub\s+)?struct\s+(\w+)\s*(?:<[^>]*>)?\s*\{', + re.MULTILINE + ) + + # Pattern for enums + enum_pattern = re.compile( + r'^(?:pub\s+)?enum\s+(\w+)\s*\{', + re.MULTILINE + ) + + # Pattern for traits + trait_pattern = re.compile( + r'^(?:pub\s+)?trait\s+(\w+)\s*\{', + re.MULTILINE + ) + + # Pattern for impl blocks + impl_pattern = re.compile( + r'^impl\s+(?:<[^>]*>)?\s*(\w+)\s*(?:<[^>]*>)?\s*\{', + re.MULTILINE + ) + + # Pattern for modules + mod_pattern = re.compile( + r'^(?:pub\s+)?mod\s+(\w+)\s*\{', + re.MULTILINE + ) + + # Parse functions + for match in func_pattern.finditer(code): + func_name = match.group(1) + start_pos = match.start() + start_line = code[:start_pos].count('\n') + end_line = _find_rust_brace_block_end(lines, start_line) + func_code = "".join(lines[start_line:end_line + 1]) + + # Enhanced function analysis + is_async = 'async' in match.group(0) + is_unsafe = 'unsafe' in match.group(0) + is_pub = 'pub' in match.group(0) + + chunk_text = f"File: {filepath}\nType: function\nName: {func_name}\n" + if is_async: + chunk_text += "Async: yes\n" + if is_unsafe: + chunk_text += "Unsafe: yes\n" + if is_pub: + chunk_text += "Visibility: pub\n" + chunk_text += f"Code:\n{func_code}" + + metadata = { + "file": str(filepath), + "name": func_name, + "type": "function", + "line": start_line + 1, + "language": "rust", + "is_async": is_async, + "is_unsafe": is_unsafe, + "visibility": "pub" if is_pub else "private" + } + + chunks.append({"text": chunk_text, "metadata": metadata}) + + # Parse structs + for match in struct_pattern.finditer(code): + struct_name = match.group(1) + start_pos = match.start() + start_line = code[:start_pos].count('\n') + end_line = _find_rust_brace_block_end(lines, start_line) + struct_code = "".join(lines[start_line:end_line + 1]) + + # Extract fields from struct + fields = _extract_rust_struct_fields(struct_code) + + chunk_text = f"File: {filepath}\nType: struct\nName: {struct_name}\n" + if fields: + chunk_text += f"Fields: {', '.join(fields)}\n" + chunk_text += f"Code:\n{struct_code}" + + metadata = { + "file": str(filepath), + "name": struct_name, + "type": "struct", + "line": start_line + 1, + "language": "rust", + "fields": fields + } + + chunks.append({"text": chunk_text, "metadata": metadata}) + + # Parse enums + for match in enum_pattern.finditer(code): + enum_name = match.group(1) + start_pos = match.start() + start_line = code[:start_pos].count('\n') + end_line = _find_rust_brace_block_end(lines, start_line) + enum_code = "".join(lines[start_line:end_line + 1]) + + # Extract variants from enum + variants = _extract_rust_enum_variants(enum_code) + + chunk_text = f"File: {filepath}\nType: enum\nName: {enum_name}\n" + if variants: + chunk_text += f"Variants: {', '.join(variants)}\n" + chunk_text += f"Code:\n{enum_code}" + + metadata = { + "file": str(filepath), + "name": enum_name, + "type": "enum", + "line": start_line + 1, + "language": "rust", + "variants": variants + } + + chunks.append({"text": chunk_text, "metadata": metadata}) + + # Parse traits + for match in trait_pattern.finditer(code): + trait_name = match.group(1) + start_pos = match.start() + start_line = code[:start_pos].count('\n') + end_line = _find_rust_brace_block_end(lines, start_line) + trait_code = "".join(lines[start_line:end_line + 1]) + + # Extract method signatures from trait + methods = _extract_rust_trait_methods(trait_code) + + chunk_text = f"File: {filepath}\nType: trait\nName: {trait_name}\n" + if methods: + chunk_text += f"Methods: {', '.join(methods)}\n" + chunk_text += f"Code:\n{trait_code}" + + metadata = { + "file": str(filepath), + "name": trait_name, + "type": "trait", + "line": start_line + 1, + "language": "rust", + "methods": methods + } + + chunks.append({"text": chunk_text, "metadata": metadata}) + + # Parse impl blocks + for match in impl_pattern.finditer(code): + impl_target = match.group(1) + start_pos = match.start() + start_line = code[:start_pos].count('\n') + end_line = _find_rust_brace_block_end(lines, start_line) + impl_code = "".join(lines[start_line:end_line + 1]) + + # Extract methods from impl block + impl_methods = _extract_rust_impl_methods(impl_code) + + chunk_text = f"File: {filepath}\nType: impl\nTarget: {impl_target}\n" + if impl_methods: + chunk_text += f"Implementation Methods: {', '.join(impl_methods)}\n" + chunk_text += f"Code:\n{impl_code}" + + metadata = { + "file": str(filepath), + "name": f"impl_{impl_target}", + "type": "impl", + "line": start_line + 1, + "language": "rust", + "target": impl_target, + "methods": impl_methods + } + + chunks.append({"text": chunk_text, "metadata": metadata}) + + return chunks + +def _find_rust_brace_block_end(lines, start_line): + """Find the closing brace for Rust code blocks.""" + brace_count = 0 + for i in range(start_line, len(lines)): + line = lines[i] + brace_count += line.count('{') + brace_count -= line.count('}') + if brace_count == 0: + return i + return len(lines) - 1 + +def _extract_rust_struct_fields(struct_code): + """Extract field names from Rust struct definition.""" + fields = [] + # Look for field patterns: field_name: Type, + field_matches = re.findall(r'(\w+)\s*:\s*[^,\n]+', struct_code) + fields.extend(field_matches) + return fields + +def _extract_rust_enum_variants(enum_code): + """Extract variant names from Rust enum definition.""" + variants = [] + # Look for variant patterns: VariantName, + variant_matches = re.findall(r'(\w+)(?:\([^)]*\))?\s*,', enum_code) + variants.extend(variant_matches) + return variants + +def _extract_rust_trait_methods(trait_code): + """Extract method names from Rust trait definition.""" + methods = [] + # Look for method signatures in trait + method_matches = re.findall(r'fn\s+(\w+)\s*\([^)]*\)', trait_code) + methods.extend(method_matches) + return methods + +def _extract_rust_impl_methods(impl_code): + """Extract method names from Rust impl block.""" + methods = [] + # Look for method implementations in impl block + method_matches = re.findall(r'fn\s+(\w+)\s*\([^)]*\)', impl_code) + methods.extend(method_matches) + return methods + +@lru_cache(maxsize=1000) +def parse_shell_file_cached(filepath_str: str) -> tuple: """Cached shell script parsing. Returns tuple for hashability.""" filepath = Path(filepath_str) try: @@ -1181,8 +1909,7 @@ def parse_shell_file_cached(filepath_str: str, file_hash: str) -> tuple: def parse_shell_file(filepath: Path) -> List[Dict]: """Enhanced shell script parsing with caching.""" - file_hash = get_file_hash(filepath) - cached_result = parse_shell_file_cached(str(filepath), file_hash) + cached_result = parse_shell_file_cached(str(filepath)) if not cached_result: return chunk_text_file(filepath, "shell") return [{"text": text, "metadata": dict(meta)} for text, meta in cached_result] @@ -1297,7 +2024,9 @@ def index_codebase(codebase_path: Path) -> Tuple[List[str], List[Dict[str, Any]] elif language == "sql": chunks = parse_sql_file(filepath) elif language == "shell": - chunks = parse_shell_file(filepath) + chunks = parse_shell_file(filepath)# + elif language == "rust": + chunks = parse_rust_file(filepath) else: chunks = chunk_text_file(filepath, language) @@ -1332,7 +2061,243 @@ def build_indexes(): logger.info("Building indexes with contextual weighting and batch embedding...") start_time = time.time() + # ------------------------------------------------- + # Step 1: Parse codebase and build enhanced graph + # ------------------------------------------------- texts, metadatas = index_codebase(CODEBASE_PATH) + + graph = LocalGraph(root_path=CODEBASE_PATH) + graph.clear() + + # Create graph nodes/edges from metadata with enhanced relationships + for m in metadatas: + fpath = m.get("file") + lang = m.get("language", "unknown") + node_type = m.get("type", "unknown") + name = m.get("name", "") + block_type = m.get("block_type", "") + component_name = m.get("component_name") + + # Ensure file node exists + file_node_id = f"file::{fpath}" + graph.add_node(file_node_id, type="File", path=fpath, lang=lang) + + # Create appropriate node based on type + if node_type in ["function", "method", "constructor"]: + # Function-like entities + if node_type == "method" and "class" in m: + # Method belongs to a class + node_id = f"{lang}::{node_type}::{fpath}::{m['class']}.{name}" + graph.add_node(node_id, type=node_type, name=name, file=fpath, + class_name=m.get("class"), line=m.get("line"), + return_type=m.get("return_type"), parameters=m.get("parameters", [])) + # Connect method to its class + class_node_id = f"{lang}::class::{fpath}::{m['class']}" + graph.add_edge(class_node_id, node_id, "contains") + + return_type = m.get("return_type") + if return_type and return_type != "void": + # Try to find return type class/interface + for type_id, type_data in graph.find_nodes(name=return_type): + graph.add_edge(node_id, type_id, "returns") + else: + # Regular function + node_id = f"{lang}::{node_type}::{fpath}::{name}" + graph.add_node(node_id, type=node_type, name=name, file=fpath, line=m.get("line"), + return_type=m.get("return_type"), parameters=m.get("parameters", [])) + graph.add_edge(file_node_id, node_id, "contains") + + elif node_type in ["class", "struct", "interface", "enum"]: + # Type definitions + node_id = f"{lang}::{node_type}::{fpath}::{name}" + graph.add_node(node_id, type=node_type, name=name, file=fpath, line=m.get("line"), + extends=m.get("extends"), implements=m.get("implements", [])) + graph.add_edge(file_node_id, node_id, "contains") + + extends_class = m.get("extends") + if extends_class: + # Try to find the parent class node + for parent_id, parent_data in graph.find_nodes(type="class", name=extends_class): + graph.add_edge(node_id, parent_id, "extends") + + implements_interfaces = m.get("implements", []) + for interface_name in implements_interfaces: + for interface_id, interface_data in graph.find_nodes(type="interface", name=interface_name): + graph.add_edge(node_id, interface_id, "implements") + + elif node_type == "field": + # Class/struct fields + node_id = f"{lang}::field::{fpath}::{name}" + graph.add_node(node_id, type=node_type, name=name, file=fpath, line=m.get("line"), + field_type=m.get("field_type")) + # If we have class context, connect to class + if "class" in m: + class_node_id = f"{lang}::class::{fpath}::{m['class']}" + graph.add_edge(class_node_id, node_id, "contains") + else: + graph.add_edge(file_node_id, node_id, "contains") + + field_type = m.get("field_type") + if field_type and field_type != "unknown": + # Try to find the field type class + for type_id, type_data in graph.find_nodes(name=field_type): + graph.add_edge(node_id, type_id, "type_reference") + + elif node_type in ["style", "markup", "code_block"]: + # Code blocks and stylistic elements + block_type = m.get("block_type", node_type) + node_id = f"{lang}::{block_type}::{fpath}::{name}" + graph.add_node(node_id, type=node_type, name=name, file=fpath, + line=m.get("line"), block_type=block_type) + graph.add_edge(file_node_id, node_id, "contains") + + else: + # Generic code element fallback + node_id = f"{lang}::{node_type}::{fpath}::{name}" + graph.add_node(node_id, type=node_type, name=name, file=fpath, line=m.get("line")) + graph.add_edge(file_node_id, node_id, "contains") + + # Handle imports if present + for imp in m.get("imports", []): + graph.add_edge(file_node_id, imp, "imports") + + # Handle function calls if present + for call in m.get("calls", []): + graph.add_edge(node_id, call, "calls") + + # Go-specific relationships + if lang == "go": + if node_type == "method": + # Connect method to its receiver type + receiver_type = m.get("receiver_base_type") + if receiver_type: + # Try to find the receiver type (struct/interface) + for type_id, type_data in graph.find_nodes(type="struct", name=receiver_type): + graph.add_edge(type_id, node_id, "has_method") + for type_id, type_data in graph.find_nodes(type="interface", name=receiver_type): + graph.add_edge(type_id, node_id, "has_method") + + elif node_type == "struct": + # Connect struct to its fields (if we had field type info) + fields = m.get("fields", []) + # Could potentially connect fields to their types if we extract field types + + elif node_type == "interface": + # Connect interface to its required methods + interface_methods = m.get("interface_methods", []) + # This helps identify what types implement this interface + + elif lang == "svelte" and component_name: + # Create component node if it doesn't exist + component_node_id = f"svelte::component::{fpath}::{component_name}" + if component_node_id not in graph.graph: + graph.add_node(component_node_id, type="component", name=component_name, + file=fpath, language="svelte") + graph.add_edge(file_node_id, component_node_id, "contains") + + # Connect different parts to the component + if node_type in ["script", "script_module", "style", "markup"]: + graph.add_edge(component_node_id, node_id, "contains") + + # Component props and interface + if node_type == "prop": + graph.add_edge(component_node_id, node_id, "has_prop") + + # Component usage relationships + if node_type == "markup": + used_components = m.get("used_components", []) + for used_comp in used_components: + # Try to find the used component in the graph + for comp_id, comp_data in graph.find_nodes(type="component", name=used_comp): + graph.add_edge(component_node_id, comp_id, "uses_component") + + # Event handler relationships + event_handlers = m.get("event_handlers", []) + for event in event_handlers: + graph.add_edge(component_node_id, f"event::{event}", "handles_event") + + # Svelte script relationships (TypeScript) + elif lang == "typescript" and block_type in ["script", "script_module"]: + # Connect TypeScript declarations to their Svelte component + if component_name: + component_node_id = f"svelte::component::{fpath}::{component_name}" + graph.add_edge(component_node_id, node_id, "contains") + + # Import relationships + for imp in m.get("imports", []): + # Try to resolve imports to other components/files + if imp.endswith('.svelte'): + imported_component = Path(imp).stem + if imported_component[0].islower(): + imported_component = imported_component[0].upper() + imported_component[1:] + # Look for the imported component + for comp_id, comp_data in graph.find_nodes(type="component", name=imported_component): + graph.add_edge(node_id, comp_id, "imports") + elif lang == "rust": + if node_type == "struct": + # Connect struct to its fields + fields = m.get("fields", []) + for field in fields: + graph.add_edge(node_id, f"rust::field::{fpath}::{name}.{field}", "has_field") + + elif node_type == "enum": + # Connect enum to its variants + variants = m.get("variants", []) + for variant in variants: + graph.add_edge(node_id, f"rust::variant::{fpath}::{name}::{variant}", "has_variant") + + elif node_type == "trait": + # Connect trait to its required methods + methods = m.get("methods", []) + for method in methods: + graph.add_edge(node_id, f"rust::trait_method::{name}::{method}", "requires_method") + + elif node_type == "impl": + # Connect impl block to its target type + target = m.get("target") + if target: + # Try to find the target struct/enum + for target_id, target_data in graph.find_nodes(name=target): + graph.add_edge(node_id, target_id, "implements_for") + + # Connect impl block to implemented traits + traits = m.get("traits", []) + for trait_name in traits: + for trait_id, trait_data in graph.find_nodes(type="trait", name=trait_name): + graph.add_edge(node_id, trait_id, "implements") + + elif node_type == "function": + # Connect function to its return type + return_type = m.get("return_type") + if return_type: + # Try to find return type in graph + for type_id, type_data in graph.find_nodes(name=return_type): + graph.add_edge(node_id, type_id, "returns") + + logger.info("Building method parameter type relationships...") + param_connections = 0 + for node_id, node_data in graph.graph.nodes(data=True): + if node_data.get('type') in ['method', 'function', 'constructor']: + parameters = node_data.get('parameters', []) + for param in parameters: + # Extract type from parameter string "TypeName paramName" + param_type = param.split()[0] if param and ' ' in param else None + if param_type and param_type not in ['void', 'int', 'String', 'boolean', 'long', 'double', 'float']: + # Try to find this type in the graph + for type_id, type_data in graph.find_nodes(name=param_type): + graph.add_edge(node_id, type_id, "uses_parameter") + param_connections += 1 + + logger.info(f"Added {param_connections} parameter type relationships") + + graph.save() + graph.to_json() + graph.to_toon() + logger.info(f"Graph built with {graph.graph.number_of_nodes()} nodes and {graph.graph.number_of_edges()} edges.") + + # ------------------------------------------------- + # Step 2: embedding/index pipeline + # ------------------------------------------------- os.environ["OLLAMA_HOST"] = OLLAMA_BASE_URL embeddings = OllamaEmbeddings(model=EMBEDDING_MODEL, base_url=OLLAMA_BASE_URL) @@ -2081,11 +3046,19 @@ Example - health_check() → { "status":"ready","total_chunks":11234,"index_age_ @mcp.tool() def search_codebase(query: str, top_k: int = 5, rerank: bool = True) -> str: - """What it does - Hybrid RAG search that returns semantically ranked snippets with file/line info from natural language queries. -When to use - For open-ended questions or unknown patterns (“how does auth work?”). Natural language and verbose queries ONLY. -When not to use - For a single symbol or exact phrase; use find_code_references then read instead to avoid a large result set. -Rerank false is good enough for 65% of searches and is VERY fast, rerank true is much slower but good enough for 95% of searches. Best to use rerank true if rerank false didn't quite help enough. -Example - search_codebase("user authentication python", top_k=5, rerank=True) → 5 relevant snippets.""" + """Hybrid RAG search: returns ranked code snippets + file/line + docstrings + graph context (cross-file deps, inheritance, callers/callees, type refs). + +Use for: Open questions ("how does auth work?"), unknown patterns, architecture exploration. Natural language queries. +Don't use for: Single symbol/exact phrase (use find_code_references or grep then read_file_lines_tool or read), exploring known entity relationships. + +rerank=False: fast, 65% accuracy | rerank=True: slower, 95% accuracy + +Output includes: code content, docs, cross_file:[calls/extends/uses@filepath], used_by:[entity@filepath], file:[same-file entities] + +Example: search_codebase("user authentication", 5, True) → 5 results with code + "cross_file:calls:ValidateToken@auth/jwt.go|used_by:setupAuth@server/routes.go" """ + graph = LocalGraph(CODEBASE_PATH) + graph.load() + try: hy = hybrid_search(query, k=max(top_k, RERANK_TOP_N)) @@ -2094,36 +3067,24 @@ Example - search_codebase("user authentication python", top_k=5, rerank=True) candidates = [t for t, m, s in hy] rr = rerank_with_ollama_enhanced(query, candidates[:RERANK_TOP_N], top_k=top_k) - # Build clean TOON format directly + # Build enhanced results with graph context results = [] for i, (chunk_text, score) in enumerate(rr, 1): meta = next((m for t, m, s in hy if t == chunk_text), {}) - results.append({ - 'file': meta.get('file', ''), - 'line': meta.get('line', ''), - 'type': meta.get('type', 'code'), - 'score': score, - 'content': _extract_clean_content(chunk_text) - }) + results.append(_build_enhanced_result(chunk_text, meta, score, graph)) - return _format_toon_results(results, query, "search") + return _format_enhanced_results(results, query, "search") except Exception as e: logger.warning(f"Rerank step failed: {e}") # Fall through to non-reranked results - # Non-reranked results - build clean TOON format directly + # Non-reranked results with graph context results = [] for i, (text, meta, score) in enumerate(hy[:top_k], 1): - results.append({ - 'file': meta.get('file', ''), - 'line': meta.get('line', ''), - 'type': meta.get('type', 'code'), - 'score': score, - 'content': _extract_clean_content(text) - }) + results.append(_build_enhanced_result(text, meta, score, graph)) - return _format_toon_results(results, query, "search") + return _format_enhanced_results(results, query, "search") except RuntimeError as e: return f"Error: {e}" @@ -2132,6 +3093,222 @@ Example - search_codebase("user authentication python", top_k=5, rerank=True) return f"Search failed: {e}" +def _build_enhanced_result(chunk_text: str, meta: dict, score: float, graph: Optional[LocalGraph] = None) -> dict: + """Build enhanced result with graph context and important metadata.""" + # Extract key metadata + file_path = meta.get('file', '') + line_num = meta.get('line', '') + entity_type = meta.get('type', 'code') + language = meta.get('language', '') + entity_name = meta.get('name', '') + + # Extract clean content and docstring + clean_content = _extract_clean_content(chunk_text) + docstring = _extract_docstring(chunk_text) + + # Get graph context for this result + graph_context = _get_result_graph_context(meta, graph) + + return { + 'file': file_path, + 'line': line_num, + 'type': entity_type, + 'language': language, + 'name': entity_name, + 'score': score, + 'content': clean_content, + 'docstring': docstring, + 'graph_context': graph_context + } + + +def _get_result_graph_context(meta: dict, graph: Optional[LocalGraph] = None) -> dict: + """Get relevant graph context for a search result with cross-file awareness.""" + if graph is None or graph.graph.number_of_nodes() == 0: + return {} + + context = {} + file_path = meta.get('file', '') + entity_name = meta.get('name', '') + entity_type = meta.get('type', '') + language = meta.get('language', '') + + # Build node ID + node_id = None + if entity_name and file_path and language: + if entity_type in ['class', 'struct', 'interface', 'enum']: + node_id = f"{language}::{entity_type}::{file_path}::{entity_name}" + elif entity_type in ['method', 'function', 'constructor']: + class_name = meta.get('class') + if class_name: + node_id = f"{language}::{entity_type}::{file_path}::{class_name}.{entity_name}" + else: + node_id = f"{language}::{entity_type}::{file_path}::{entity_name}" + elif entity_type == 'component' and language == 'svelte': + node_id = f"svelte::component::{file_path}::{entity_name}" + + if node_id and node_id in graph.graph: + # Get direct relationships WITH file info + neighbors = list(graph.neighbors(node_id)) + if neighbors: + context['relationships'] = [] + context['cross_file_deps'] = [] # NEW: Track cross-file dependencies + + for neighbor_id, neighbor_data in neighbors[:10]: # Show more neighbors + edge_data = graph.graph.edges[node_id, neighbor_id] + rel_type = edge_data.get('type', 'related') + neighbor_name = neighbor_data.get('name', neighbor_id.split('::')[-1]) + neighbor_file = neighbor_data.get('file', '') + neighbor_type = neighbor_data.get('type', '') + + # Build compact relationship string with file info + if neighbor_file and neighbor_file != file_path: + # Cross-file relationship - show file + rel_str = f"{rel_type}:{neighbor_name}@{neighbor_file}" + context['cross_file_deps'].append(rel_str) + else: + # Same-file relationship - omit file for brevity + rel_str = f"{rel_type}:{neighbor_name}" + + context['relationships'].append(rel_str) + + # NEW: Find reverse dependencies (what depends on THIS entity) + incoming = [] + for predecessor in graph.graph.predecessors(node_id): + pred_data = graph.graph.nodes[predecessor] + pred_file = pred_data.get('file', '') + pred_name = pred_data.get('name', '') + edge_data = graph.graph.edges[predecessor, node_id] + rel_type = edge_data.get('type', 'related') + + # Only show cross-file incoming dependencies + if pred_file and pred_file != file_path: + incoming.append(f"{rel_type}:{pred_name}@{pred_file}") + + if incoming: + context['used_by'] = incoming[:10] + + # Get file-level context (same as before) + file_node_id = f"file::{file_path}" + if file_node_id in graph.graph: + file_neighbors = list(graph.neighbors(file_node_id)) + if file_neighbors: + context['file_entities'] = [] + for neighbor_id, neighbor_data in file_neighbors[:5]: + neighbor_type = neighbor_data.get('type', '') + neighbor_name = neighbor_data.get('name', '') + if neighbor_type and neighbor_name and neighbor_name != entity_name: + context['file_entities'].append(f"{neighbor_type}:{neighbor_name}") + + return context + + +def _extract_docstring(chunk_text: str) -> str: + """Extract docstring or comments from chunk text.""" + lines = chunk_text.split('\n') + doc_lines = [] + + # Look for docstring patterns + in_docstring = False + for line in lines: + stripped = line.strip() + + # Java/Go style comments + if stripped.startswith('//') or stripped.startswith('/*') or stripped.startswith('*'): + doc_lines.append(stripped) + # Python style docstrings + elif '"""' in line or "'''" in line: + if not in_docstring: + in_docstring = True + else: + in_docstring = False + break + elif in_docstring: + doc_lines.append(stripped) + # Specific doc patterns in the metadata section + elif stripped.startswith('Doc:') and len(stripped) > 4: + doc_content = stripped[4:].strip() + if doc_content and doc_content not in ['None', '""']: + doc_lines.append(doc_content) + + # Clean up docstring + if doc_lines: + docstring = ' '.join(doc_lines) + docstring = re.sub(r'\s+', ' ', docstring) + return docstring.strip()[:300] # Reasonable limit + + return "" + + +def _format_enhanced_results(results: List[Dict], query: str, result_type: str) -> str: + """Format enhanced results with dense LLM-optimized context.""" + if not results: + return f"# {result_type.title()}: {query}\nNo results found.\n" + + lines = [ + f"# {result_type.title()}: {query}", + f"results[{len(results)}]{{file,line,type,name,score,content,context}}:" + ] + + for result in results: + context_parts = [] + + # Graph context as dense key:value pairs + graph_context = result.get('graph_context', {}) + + # Cross-file dependencies (highest value) + if graph_context.get('cross_file_deps'): + deps = '|'.join(graph_context['cross_file_deps'][:8]) + context_parts.append(f"xfile:{deps}") + + # All relationships + if graph_context.get('relationships'): + rels = '|'.join(graph_context['relationships'][:12]) + context_parts.append(f"rels:{rels}") + + # Reverse dependencies + if graph_context.get('used_by'): + used = '|'.join(graph_context['used_by'][:6]) + context_parts.append(f"used:{used}") + + # File context + if graph_context.get('file_entities'): + entities = '|'.join(graph_context['file_entities'][:10]) + context_parts.append(f"file:{entities}") + + # Docstring (if valuable) + docstring = result.get('docstring', '') + if docstring and len(docstring) > 10: # Only include substantial docs + context_parts.append(f"doc:{docstring[:200]}") + + context_str = ";".join(context_parts) + + # Minimal row with essential info + row = [ + result.get('file', ''), + result.get('line', ''), + result.get('type', 'code'), + result.get('name', ''), + f"{result.get('score', 0.0):.2f}", + result.get('content', '')[:500], # More content tokens + context_str + ] + + # Efficient escaping + escaped_row = [] + for field in row: + field_str = str(field) + if ',' in field_str or ';' in field_str: + escaped = field_str.replace('"', '\\"') + escaped_row.append(f'"{escaped}"') + else: + escaped_row.append(field_str) + + lines.append(" " + ",".join(escaped_row)) + + return "\n".join(lines) + + def _extract_clean_content(chunk_text: str) -> str: """Extract clean content from chunk text by removing duplicate metadata.""" lines = chunk_text.split('\n') @@ -2157,36 +3334,6 @@ def _extract_clean_content(chunk_text: str) -> str: return content.strip() -def _format_toon_results(results: List[Dict], query: str, result_type: str) -> str: - """Format results in clean TOON format.""" - if not results: - return f"# {result_type.title()}: {query}\nNo results found.\n" - - lines = [f"# {result_type.title()}: {query}", f"results[{len(results)}]{{file,line,score,content}}:"] - - for result in results: - row = [ - result.get('file', ''), - result.get('line', ''), - f"{result.get('score', 0.0):.2f}", - result.get('content', '')[:500] # Reasonable limit - ] - - # Escape fields if needed - escaped_row = [] - for field in row: - field_str = str(field) - if any(char in field_str for char in [',', '"', '\n', '\r']): - escaped = field_str.replace('"', '\\"') - escaped_row.append(f'"{escaped}"') - else: - escaped_row.append(field_str) - - lines.append(" " + ",".join(escaped_row)) - - return "\n".join(lines) - - @mcp.tool() def find_code_references(symbol: str, top_k: int = 20) -> str: """What it does - Fast, unranked lookup of all file/line occurrences of a symbol. @@ -2337,8 +3484,8 @@ def init_repo(git_url: str) -> str: def rebuild_index() -> str: """What it does - Re-creates all embeddings, BM25, and metadata indexes after major code changes. When to use - When the codebase has been pulled or refactored and you suspect stale search results, and only after explicit instruction to do so. -When not to use - On every query; it’s expensive and unnecessary if the index is already up-to-date. -Example - rebuild_index() → '✅ Index rebuilt (13.2 s); 11,234 chunks'""" +When not to use - On every query; it's expensive and unnecessary if the index is already up-to-date. +Example - rebuild_index() → '✅ Index rebuilt (13.2 s); 11,234 chunks'""" with _startup_lock: try: # Clear LRU caches @@ -2349,6 +3496,8 @@ Example - rebuild_index() → '✅ Index rebuilt (13.2 s); 11,234 chunks'""" parse_svelte_file_cached.cache_clear() parse_shell_file_cached.cache_clear() + LocalGraph(root_path=CODEBASE_PATH).clear() + t0 = time.time() build_indexes() dt = time.time() - t0 diff --git a/requirements.txt b/requirements.txt index ac157f6..ef14456 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,3 +26,5 @@ psutil==5.9.5 # --- SQL parsing stack --- sqlglot==25.3.0 # Portable SQL AST + transpiler (fast for SELECTs, DML) sqlparse==0.5.1 # Lightweight fallback tokenizer (simple/heuristic parsing) + +networkx==3.1 # Graph library for dependency graph \ No newline at end of file diff --git a/tools/parse_rust_ast.rs b/tools/parse_rust_ast.rs new file mode 100644 index 0000000..0708b2d --- /dev/null +++ b/tools/parse_rust_ast.rs @@ -0,0 +1,315 @@ +// Build from project root: +// rustc -O -o tools/parse_rust_ast tools/parse_rust_ast.rs + +use std::env; +use std::fs; +use std::process; +use syn::{FnArg, Item, ItemEnum, ItemFn, ItemImpl, ItemStruct, ItemTrait, Pat, Type, Visibility}; + +#[derive(serde::Serialize)] +struct RustDecl { + name: String, + type_: String, + start_line: usize, + end_line: usize, + visibility: String, + is_async: bool, + is_unsafe: bool, + generics: Vec, + traits: Vec, + fields: Vec, + methods: Vec, + return_type: Option, + parameters: Vec, +} + +fn main() { + let args: Vec = env::args().collect(); + if args.len() < 2 { + eprintln!("Usage: {} ", args[0]); + process::exit(1); + } + + let filename = &args[1]; + let content = match fs::read_to_string(filename) { + Ok(content) => content, + Err(e) => { + eprintln!("ERR: Failed to read file {}: {}", filename, e); + process::exit(2); + } + }; + + let syntax = match syn::parse_file(&content) { + Ok(syntax) => syntax, + Err(e) => { + eprintln!("ERR: Failed to parse Rust file {}: {}", filename, e); + process::exit(3); + } + }; + + let mut decls = Vec::new(); + + for item in syntax.items { + match item { + Item::Fn(item_fn) => { + if let Some(decl) = parse_function(item_fn, &content) { + decls.push(decl); + } + } + Item::Struct(item_struct) => { + if let Some(decl) = parse_struct(item_struct, &content) { + decls.push(decl); + } + } + Item::Enum(item_enum) => { + if let Some(decl) = parse_enum(item_enum, &content) { + decls.push(decl); + } + } + Item::Trait(item_trait) => { + if let Some(decl) = parse_trait(item_trait, &content) { + decls.push(decl); + } + } + Item::Impl(item_impl) => { + if let Some(decl) = parse_impl(item_impl, &content) { + decls.push(decl); + } + } + Item::Mod(item_mod) => { + if let Some(decl) = parse_mod(item_mod, &content) { + decls.push(decl); + } + } + _ => {} + } + } + + let output = serde_json::to_string(&decls).unwrap(); + println!("{}", output); +} + +fn parse_function(item_fn: ItemFn, content: &str) -> Option { + let start_line = item_fn.span().start().line; + let end_line = item_fn.span().end().line; + + let mut parameters = Vec::new(); + for input in &item_fn.sig.inputs { + if let FnArg::Typed(pat_type) = input { + if let Pat::Ident(pat_ident) = &*pat_type.pat { + parameters.push(pat_ident.ident.to_string()); + } + } + } + + let return_type = match &item_fn.sig.output { + syn::ReturnType::Default => None, + syn::ReturnType::Type(_, ty) => Some(type_to_string(ty)), + }; + + Some(RustDecl { + name: item_fn.sig.ident.to_string(), + type_: "function".to_string(), + start_line, + end_line, + visibility: visibility_to_string(&item_fn.vis), + is_async: item_fn.sig.asyncness.is_some(), + is_unsafe: item_fn.sig.unsafety.is_some(), + generics: item_fn + .sig + .generics + .params + .iter() + .map(|param| param.to_string()) + .collect(), + traits: Vec::new(), + fields: Vec::new(), + methods: Vec::new(), + return_type, + parameters, + }) +} + +fn parse_struct(item_struct: ItemStruct, content: &str) -> Option { + let start_line = item_struct.span().start().line; + let end_line = item_struct.span().end().line; + + let fields: Vec = item_struct + .fields + .iter() + .filter_map(|field| field.ident.as_ref().map(|ident| ident.to_string())) + .collect(); + + Some(RustDecl { + name: item_struct.ident.to_string(), + type_: "struct".to_string(), + start_line, + end_line, + visibility: visibility_to_string(&item_struct.vis), + is_async: false, + is_unsafe: false, + generics: item_struct + .generics + .params + .iter() + .map(|param| param.to_string()) + .collect(), + traits: Vec::new(), + fields, + methods: Vec::new(), + return_type: None, + parameters: Vec::new(), + }) +} + +fn parse_enum(item_enum: ItemEnum, content: &str) -> Option { + let start_line = item_enum.span().start().line; + let end_line = item_enum.span().end().line; + + let variants: Vec = item_enum + .variants + .iter() + .map(|variant| variant.ident.to_string()) + .collect(); + + Some(RustDecl { + name: item_enum.ident.to_string(), + type_: "enum".to_string(), + start_line, + end_line, + visibility: visibility_to_string(&item_enum.vis), + is_async: false, + is_unsafe: false, + generics: item_enum + .generics + .params + .iter() + .map(|param| param.to_string()) + .collect(), + traits: Vec::new(), + fields: variants, + methods: Vec::new(), + return_type: None, + parameters: Vec::new(), + }) +} + +fn parse_trait(item_trait: ItemTrait, content: &str) -> Option { + let start_line = item_trait.span().start().line; + let end_line = item_trait.span().end().line; + + let methods: Vec = item_trait + .items + .iter() + .filter_map(|item| { + if let syn::TraitItem::Fn(method) = item { + Some(method.sig.ident.to_string()) + } else { + None + } + }) + .collect(); + + Some(RustDecl { + name: item_trait.ident.to_string(), + type_: "trait".to_string(), + start_line, + end_line, + visibility: visibility_to_string(&item_trait.vis), + is_async: false, + is_unsafe: false, + generics: item_trait + .generics + .params + .iter() + .map(|param| param.to_string()) + .collect(), + traits: Vec::new(), + fields: Vec::new(), + methods, + return_type: None, + parameters: Vec::new(), + }) +} + +fn parse_impl(item_impl: ItemImpl, content: &str) -> Option { + let start_line = item_impl.span().start().line; + let end_line = item_impl.span().end().line; + + let target = type_to_string(&item_impl.self_ty); + let methods: Vec = item_impl + .items + .iter() + .filter_map(|item| { + if let syn::ImplItem::Fn(method) = item { + Some(method.sig.ident.to_string()) + } else { + None + } + }) + .collect(); + + let traits: Vec = item_impl + .trait_ + .as_ref() + .map(|(_, path, _)| path_to_string(path)) + .into_iter() + .collect(); + + Some(RustDecl { + name: format!("impl_{}", target), + type_: "impl".to_string(), + start_line, + end_line, + visibility: "".to_string(), + is_async: false, + is_unsafe: item_impl.unsafety.is_some(), + generics: item_impl + .generics + .params + .iter() + .map(|param| param.to_string()) + .collect(), + traits, + fields: Vec::new(), + methods, + return_type: None, + parameters: Vec::new(), + }) +} + +fn parse_mod(item_mod: syn::ItemMod, content: &str) -> Option { + let start_line = item_mod.span().start().line; + let end_line = item_mod.span().end().line; + + Some(RustDecl { + name: item_mod.ident.to_string(), + type_: "module".to_string(), + start_line, + end_line, + visibility: visibility_to_string(&item_mod.vis), + is_async: false, + is_unsafe: false, + generics: Vec::new(), + traits: Vec::new(), + fields: Vec::new(), + methods: Vec::new(), + return_type: None, + parameters: Vec::new(), + }) +} + +fn visibility_to_string(vis: &Visibility) -> String { + match vis { + Visibility::Public(_) => "pub".to_string(), + _ => "private".to_string(), + } +} + +fn type_to_string(ty: &Type) -> String { + format!("{}", quote::quote!(#ty)) +} + +fn path_to_string(path: &syn::Path) -> String { + format!("{}", quote::quote!(#path)) +}