diff --git a/mcp_codebase.py b/mcp_codebase.py index 7ccd27f..a4b27a8 100644 --- a/mcp_codebase.py +++ b/mcp_codebase.py @@ -787,153 +787,239 @@ def include_leading_comments(lines: List[str], stmt_start_line: int, max_context @lru_cache(maxsize=1000) def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: - """Cached Go AST parsing with enhanced metadata for graph relationships.""" + """Cached Go parsing with enhanced metadata for graph relationships. + ALWAYS returns a tuple of (text, metadata_tuple) entries. If the Go helper + fails or produces no chunks, return a single fallback chunk containing the + whole file text and minimal metadata. + """ filepath = Path(filepath_str) chunks = [] - helper = Path("./tools/parse_go_ast") - if helper.exists(): - try: - proc = subprocess.run( - [str(helper), str(filepath)], - capture_output=True, - text=True, - check=True, - timeout=20 - ) - decls = json.loads(proc.stdout) - lines = filepath.read_text(encoding="utf-8").splitlines(keepends=True) + logger.info(f"🐹 Parsing Go file: {filepath}") - for d in decls: - # Use lowercase keys (matching the Go JSON output) - start = max(0, d.get("start_line", 1) - 1) - end = d.get("end_line", start + 1) - name = d.get("name", "") - typ = d.get("type", "") - doc_comment = d.get("doc_comment", "") - receiver = d.get("receiver", "") - fields = d.get("fields", []) - methods = d.get("methods", []) + go_helper = Path("./tools/parse_go_ast") + if not go_helper.is_file(): + logger.warning(f"Go parser binary not found: {go_helper}. Using fallback regex parsing for {filepath}") + return _parse_go_fallback(filepath_str, file_hash) - # Skip package declarations - if typ == "package": - continue - - # Build display type - display_type = typ - if receiver: - display_type = f"method ({receiver})" - - chunk_code = "".join(lines[start:end]) - - chunk_text = f"File: {filepath}\nType: {display_type}\nName: {name}\n" - if doc_comment: - chunk_text += f"Doc:\n{doc_comment}\n\n" - if fields: - chunk_text += f"Fields: {', '.join(fields)}\n" - if methods: - chunk_text += f"Methods: {', '.join(methods)}\n" - chunk_text += f"Code:\n{chunk_code}" - - # Build comprehensive metadata - metadata = { - "file": str(filepath), - "name": name, - "type": typ, - "receiver": receiver, - "line": start + 1, - "language": "go", - } - - # Enhanced metadata for graph relationships - if fields: - metadata["fields"] = fields - if methods: - metadata["interface_methods"] = methods - if receiver: - metadata["receiver_type"] = receiver - # Extract receiver type without pointer for type matching - receiver_base = receiver.replace('*', '') - metadata["receiver_base_type"] = receiver_base - - chunks.append({ - "text": chunk_text, - "metadata": metadata - }) - - if chunks: # If helper worked, return its results - return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) - - except subprocess.CalledProcessError as e: - logger.warning(f"parse_go_ast failed for {filepath}: {e.stderr}; falling back to regex") - except json.JSONDecodeError as e: - logger.warning(f"parse_go_ast output invalid JSON for {filepath}: {e}; falling back to regex") - except Exception as e: - logger.warning(f"parse_go_file helper failed for {filepath}: {e}; falling back to regex") - - # Fallback: Use regex-based parsing try: - content = filepath.read_text(encoding="utf-8") - lines = content.splitlines(keepends=True) + code = filepath.read_text(encoding="utf-8") + lines = code.splitlines(keepends=True) - # Pattern to match Go function declarations - func_pattern = re.compile( - r'^func\s+(?:\([^)]+\)\s+)?(\w+)\s*\([^)]*\)(?:\s*\([^)]*\)|\s+[\w\[\].*]+)?\s*\{', - re.MULTILINE + proc = subprocess.run( + [str(go_helper), str(filepath)], + capture_output=True, + text=True, + check=True, + timeout=20 ) + decls = json.loads(proc.stdout) + logger.info(f" AST helper found {len(decls)} declarations") - for match in func_pattern.finditer(content): - func_name = match.group(1) - start_pos = match.start() - start_line = content[:start_pos].count('\n') - end_line = find_brace_block_end_go(lines, start_line) - func_code = "".join(lines[start_line:end_line + 1]) - comments = extract_go_comments(lines, start_line) + for i, d in enumerate(decls): + # Use lowercase keys (matching the Go JSON output) + start_line = max(0, d.get("start_line", 1) - 1) + end_line = d.get("end_line", start_line + 1) - # Try to extract receiver type from function signature - receiver_type = "" - func_sig = content[start_pos:match.end()] - receiver_match = re.search(r'func\s*\(([^)]+)\)', func_sig) - if receiver_match: - receiver_part = receiver_match.group(1) - # Extract type from receiver (e.g., "u *User" -> "*User") - receiver_type_match = re.search(r'\*?(\w+)', receiver_part.split()[-1] if ' ' in receiver_part else receiver_part) - if receiver_type_match: - receiver_type = receiver_type_match.group(0) + # Guard the end_line to be at least start_line+1 + if end_line <= start_line: + end_line = start_line + 1 - chunk_text = f"File: {filepath}\nType: function\nName: {func_name}\n" - if comments: - chunk_text += f"Comments:\n{comments}\n\n" - chunk_text += f"Code:\n{func_code}" + # Make slice safe even if end_line > len(lines) + chunk_code = "".join(lines[start_line:end_line]) + + name = d.get("name", "") + typ = d.get("type", "unknown") + doc_comment = d.get("doc_comment", "") + receiver = d.get("receiver", "") + full_name = d.get("full_name", name) + fields = d.get("fields", []) + methods = d.get("methods", []) + + # Skip package declarations (they don't contribute to code graph) + if typ == "package": + continue + + # Skip empty or invalid declarations + if not name or typ == "unknown": + continue metadata = { "file": str(filepath), - "name": func_name, - "type": "function", + "name": name, + "type": typ, "line": start_line + 1, "language": "go", } - # Receiver type for methods - if receiver_type: - metadata["receiver_type"] = receiver_type - metadata["receiver_base_type"] = receiver_type.replace('*', '') - metadata["type"] = "method" + # Build comprehensive chunk text + chunk_text = f"File: {filepath}\nType: {typ}\nName: {name}\n" + + # Add receiver information for methods + if receiver: + metadata["receiver"] = receiver + # Extract receiver type without pointer for type matching + receiver_base = receiver.replace('*', '').strip() + metadata["receiver_base_type"] = receiver_base + chunk_text += f"Receiver: {receiver}\n" + + # Add full name if different from name (methods have qualified names) + if full_name and full_name != name: + metadata["full_name"] = full_name + + # Add documentation + if doc_comment: + metadata["docstring"] = doc_comment + chunk_text += f"Doc:\n{doc_comment}\n\n" + + # Add fields for structs + if fields: + metadata["fields"] = fields + chunk_text += f"Fields: {', '.join(fields)}\n" + + # Add methods for interfaces + if methods: + metadata["interface_methods"] = methods + chunk_text += f"Methods: {', '.join(methods)}\n" + + chunk_text += f"Code:\n{chunk_code}" + + logger.info(f" Declaration {i+1}: {typ} {name}") chunks.append({ "text": chunk_text, "metadata": metadata }) - # Structs with field extraction + except subprocess.CalledProcessError as e: + stderr = e.stderr if e.stderr else "unknown error" + logger.warning(f"Go AST helper failed for {filepath}: {stderr}") + except json.JSONDecodeError as e: + logger.warning(f"Go AST helper output invalid JSON for {filepath}: {e}") + except Exception as e: + logger.warning(f"Go AST helper failed for {filepath}: {e}") + + # If parser produced chunks, return them in the (text, metadata_tuple) format + if chunks: + logger.info(f"✅ Successfully parsed {len(chunks)} chunks from {filepath}") + return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) + + # Fallback to regex-based parsing + logger.warning(f"⚠️ Go AST helper produced no chunks for {filepath}. Trying regex fallback.") + return _parse_go_fallback(filepath_str, file_hash) + + +def _parse_go_fallback(filepath_str: str, file_hash: str) -> tuple: + """Fallback regex-based Go parser when AST helper fails or is unavailable.""" + filepath = Path(filepath_str) + chunks = [] + + logger.info(f" Using regex-based Go parsing for {filepath}") + + try: + content = filepath.read_text(encoding="utf-8") + lines = content.splitlines(keepends=True) + + # Helper function to find matching closing brace + def find_brace_block_end(start_line: int) -> int: + """Find the line with the matching closing brace.""" + brace_count = 0 + for i in range(start_line, len(lines)): + for char in lines[i]: + if char == '{': + brace_count += 1 + elif char == '}': + brace_count -= 1 + if brace_count == 0: + return i + 1 # Include closing brace line + return len(lines) + + # Helper function to extract preceding comments + def extract_comments(start_line: int) -> str: + """Extract comments immediately before a declaration.""" + comments = [] + for i in range(max(0, start_line - 10), start_line): + line = lines[i].strip() + if line.startswith('//'): + comments.append(line[2:].strip()) + elif line.startswith('/*'): + # Multi-line comment + comment_lines = [] + for j in range(i, start_line): + comment_lines.append(lines[j].strip()) + if '*/' in lines[j]: + break + comment_text = ' '.join(comment_lines) + comment_text = comment_text.replace('/*', '').replace('*/', '').strip() + comments.append(comment_text) + elif line and not line.startswith('package') and not line.startswith('import'): + # Non-empty, non-comment line before declaration - stop looking + break + return '\n'.join(comments) if comments else "" + + # Pattern to match Go function declarations (including methods) + func_pattern = re.compile( + r'^func\s+(?:\(([^)]+)\)\s+)?(\w+)\s*\([^)]*\)(?:\s*\([^)]*\)|\s+[\w\[\].*]+)?\s*\{', + re.MULTILINE + ) + + for match in func_pattern.finditer(content): + receiver_part = match.group(1) # Receiver (optional) + func_name = match.group(2) + start_pos = match.start() + start_line = content[:start_pos].count('\n') + end_line = find_brace_block_end(start_line) + func_code = "".join(lines[start_line:end_line]) + comments = extract_comments(start_line) + + # Parse receiver if present + receiver_type = "" + receiver_base_type = "" + if receiver_part: + # Extract type from receiver (e.g., "u *User" -> "*User") + parts = receiver_part.strip().split() + if len(parts) >= 2: + receiver_type = parts[-1] + elif len(parts) == 1: + receiver_type = parts[0] + receiver_base_type = receiver_type.replace('*', '').strip() + + metadata = { + "file": str(filepath), + "name": func_name, + "type": "method" if receiver_type else "function", + "line": start_line + 1, + "language": "go", + } + + chunk_text = f"File: {filepath}\nType: {metadata['type']}\nName: {func_name}\n" + + # Add receiver information for methods + if receiver_type: + metadata["receiver"] = receiver_type + metadata["receiver_base_type"] = receiver_base_type + chunk_text += f"Receiver: {receiver_type}\n" + + if comments: + metadata["docstring"] = comments + chunk_text += f"Comments:\n{comments}\n\n" + + chunk_text += f"Code:\n{func_code}" + + chunks.append({ + "text": chunk_text, + "metadata": metadata + }) + + # Pattern to match struct declarations struct_pattern = re.compile(r'^type\s+(\w+)\s+struct\s*\{', re.MULTILINE) for match in struct_pattern.finditer(content): struct_name = match.group(1) start_pos = match.start() start_line = content[:start_pos].count('\n') - end_line = find_brace_block_end_go(lines, start_line) - struct_code = "".join(lines[start_line:end_line + 1]) - comments = extract_go_comments(lines, start_line) + end_line = find_brace_block_end(start_line) + struct_code = "".join(lines[start_line:end_line]) + comments = extract_comments(start_line) # Extract field names from struct field_names = [] @@ -941,16 +1027,21 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: if struct_body_match: field_lines = struct_body_match.group(1).split('\n') for line in field_lines: - field_match = re.match(r'\s*(\w+)\s+', line.strip()) + line = line.strip() + if not line or line.startswith('//'): + continue + # Match field declarations: FieldName Type or FieldName, FieldName2 Type + field_match = re.match(r'(\w+(?:\s*,\s*\w+)*)\s+[\w\[\]*.]+', line) if field_match: - field_names.append(field_match.group(1)) - - chunk_text = f"File: {filepath}\nType: struct\nName: {struct_name}\n" - if comments: - chunk_text += f"Comments:\n{comments}\n\n" - if field_names: - chunk_text += f"Fields: {', '.join(field_names)}\n" - chunk_text += f"Code:\n{struct_code}" + # Handle multiple fields on one line + field_str = field_match.group(1) + fields = [f.strip() for f in field_str.split(',')] + field_names.extend(fields) + else: + # Check for embedded fields (just a type, no name) + embedded_match = re.match(r'(\*?[\w.]+)\s*(?://|$)', line) + if embedded_match and not line.startswith('type'): + field_names.append(f"embedded:{embedded_match.group(1)}") metadata = { "file": str(filepath), @@ -960,24 +1051,32 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: "language": "go", } - # Struct fields for graph relationships + chunk_text = f"File: {filepath}\nType: struct\nName: {struct_name}\n" + + if comments: + metadata["docstring"] = comments + chunk_text += f"Comments:\n{comments}\n\n" + if field_names: metadata["fields"] = field_names + chunk_text += f"Fields: {', '.join(field_names)}\n" + + chunk_text += f"Code:\n{struct_code}" chunks.append({ "text": chunk_text, "metadata": metadata }) - # Interfaces with method extraction + # Pattern to match interface declarations interface_pattern = re.compile(r'^type\s+(\w+)\s+interface\s*\{', re.MULTILINE) for match in interface_pattern.finditer(content): interface_name = match.group(1) start_pos = match.start() start_line = content[:start_pos].count('\n') - end_line = find_brace_block_end_go(lines, start_line) - interface_code = "".join(lines[start_line:end_line + 1]) - comments = extract_go_comments(lines, start_line) + end_line = find_brace_block_end(start_line) + interface_code = "".join(lines[start_line:end_line]) + comments = extract_comments(start_line) # Extract method signatures from interface method_names = [] @@ -985,17 +1084,14 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: if interface_body_match: method_lines = interface_body_match.group(1).split('\n') for line in method_lines: - method_match = re.match(r'\s*(\w+)\s*\([^)]*\)', line.strip()) + line = line.strip() + if not line or line.startswith('//'): + continue + # Match method declarations: MethodName(params) returnType + method_match = re.match(r'(\w+)\s*\([^)]*\)', line) if method_match: method_names.append(method_match.group(1)) - chunk_text = f"File: {filepath}\nType: interface\nName: {interface_name}\n" - if comments: - chunk_text += f"Comments:\n{comments}\n\n" - if method_names: - chunk_text += f"Methods: {', '.join(method_names)}\n" - chunk_text += f"Code:\n{interface_code}" - metadata = { "file": str(filepath), "name": interface_name, @@ -1004,19 +1100,145 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: "language": "go", } - # Interface methods for graph relationships + chunk_text = f"File: {filepath}\nType: interface\nName: {interface_name}\n" + + if comments: + metadata["docstring"] = comments + chunk_text += f"Comments:\n{comments}\n\n" + if method_names: metadata["interface_methods"] = method_names + chunk_text += f"Methods: {', '.join(method_names)}\n" + + chunk_text += f"Code:\n{interface_code}" chunks.append({ "text": chunk_text, "metadata": metadata }) + # Pattern to match type aliases and other type declarations + type_alias_pattern = re.compile(r'^type\s+(\w+)\s+(?!struct|interface)(.+?)(?:\n|$)', re.MULTILINE) + for match in type_alias_pattern.finditer(content): + type_name = match.group(1) + type_def = match.group(2).strip() + start_pos = match.start() + start_line = content[:start_pos].count('\n') + + # For simple type aliases, just capture the line + type_code = lines[start_line] if start_line < len(lines) else match.group(0) + comments = extract_comments(start_line) + + metadata = { + "file": str(filepath), + "name": type_name, + "type": "type", + "line": start_line + 1, + "language": "go", + } + + chunk_text = f"File: {filepath}\nType: type alias\nName: {type_name}\n" + + if comments: + metadata["docstring"] = comments + chunk_text += f"Comments:\n{comments}\n\n" + + chunk_text += f"Definition: {type_def}\n" + chunk_text += f"Code:\n{type_code}" + + chunks.append({ + "text": chunk_text, + "metadata": metadata + }) + + # Pattern to match const and var declarations + const_var_pattern = re.compile(r'^(const|var)\s+(?:\(|(\w+))', re.MULTILINE) + for match in const_var_pattern.finditer(content): + decl_type = match.group(1) # 'const' or 'var' + single_name = match.group(2) # Name if single declaration + start_pos = match.start() + start_line = content[:start_pos].count('\n') + + if single_name: + # Single declaration + var_code = lines[start_line] if start_line < len(lines) else match.group(0) + comments = extract_comments(start_line) + + metadata = { + "file": str(filepath), + "name": single_name, + "type": decl_type, + "line": start_line + 1, + "language": "go", + } + + chunk_text = f"File: {filepath}\nType: {decl_type}\nName: {single_name}\n" + + if comments: + metadata["docstring"] = comments + chunk_text += f"Comments:\n{comments}\n\n" + + chunk_text += f"Code:\n{var_code}" + + chunks.append({ + "text": chunk_text, + "metadata": metadata + }) + else: + # Block declaration (const ( ... ) or var ( ... )) + end_line = find_brace_block_end(start_line) if '(' in lines[start_line] else start_line + 1 + # For simplicity in fallback, just capture the whole block + block_code = "".join(lines[start_line:end_line]) + comments = extract_comments(start_line) + + # Extract individual names from block + names = re.findall(r'^\s*(\w+)\s+', block_code, re.MULTILINE) + + if names: + for name in names: + metadata = { + "file": str(filepath), + "name": name, + "type": decl_type, + "line": start_line + 1, + "language": "go", + } + + chunk_text = f"File: {filepath}\nType: {decl_type}\nName: {name}\n" + + if comments: + metadata["docstring"] = comments + + chunk_text += f"Code:\n{block_code}" + + chunks.append({ + "text": chunk_text, + "metadata": metadata + }) + except Exception as e: logger.warning(f"Regex-based Go parsing failed for {filepath}: {e}") - return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) + # Return parsed chunks or final fallback + if chunks: + logger.info(f"✅ Regex fallback parsed {len(chunks)} chunks from {filepath}") + return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) + + # Final fallback: emit a whole-file chunk + logger.warning(f"❌ No chunks created from Go parsing for {filepath}. Emitting fallback whole-file chunk.") + try: + full_code = filepath.read_text(encoding="utf-8") + except Exception: + full_code = "" + + fallback_meta = { + "file": str(filepath), + "name": filepath.name, + "type": "file", + "line": 1, + "language": "go", + } + return ((full_code, tuple(fallback_meta.items())),) def find_brace_block_end_go(lines: List[str], start_line: int) -> int: @@ -1102,16 +1324,36 @@ def parse_go_file(filepath: Path) -> List[Dict]: # Java parsing (javalang + manual body extraction) with caching # ----------------------------- @lru_cache(maxsize=1000) -def parse_java_file_cached(filepath_str: str) -> tuple: - """Cached Java parsing. Returns tuple for hashability.""" +def parse_java_file_cached(filepath_str: str, file_hash: str) -> tuple: + """Cached Java parsing with enhanced metadata for graph relationships. + ALWAYS returns a tuple of (text, metadata_tuple) entries. If parsing + fails or produces no chunks, return a single fallback chunk containing + the whole file text and minimal metadata. + """ filepath = Path(filepath_str) chunks = [] + + logger.info(f"☕ Parsing Java file: {filepath}") + try: code = filepath.read_text(encoding="utf-8") lines = code.splitlines(keepends=True) - tree = javalang.parse.parse(code) + + try: + tree = javalang.parse.parse(code) + except javalang.parser.JavaSyntaxError as e: + logger.warning(f"Java syntax error in {filepath}: {e}. Using fallback chunk.") + fallback_meta = { + "file": str(filepath), + "name": filepath.name, + "type": "file", + "line": 1, + "language": "java", + } + return ((code, tuple(fallback_meta.items())),) def find_brace_block_end(start_idx: int) -> int: + """Find the closing brace for a block starting at start_idx.""" brace_count = 0 for i in range(start_idx, len(lines)): for char in lines[i]: @@ -1120,174 +1362,423 @@ def parse_java_file_cached(filepath_str: str) -> tuple: elif char == '}': brace_count -= 1 if brace_count == 0: - return i - return len(lines) - 1 + return i + 1 # Include the closing brace line + return len(lines) # If no closing brace found, take rest of file + + def extract_docstring(node) -> str | None: + """Extract documentation/Javadoc from a node.""" + if hasattr(node, 'documentation') and node.documentation: + return node.documentation.strip() + return None + + def get_visibility(modifiers) -> str: + """Extract visibility from modifiers set.""" + if not modifiers: + return "package-private" + if 'public' in modifiers: + return "public" + if 'protected' in modifiers: + return "protected" + if 'private' in modifiers: + return "private" + return "package-private" + + def extract_imports(tree) -> list[str]: + """Extract all imports from the compilation unit.""" + imports = [] + if hasattr(tree, 'imports') and tree.imports: + for imp in tree.imports: + imports.append(imp.path) + return imports + + def extract_annotations(node) -> list[str]: + """Extract annotations from a node.""" + annotations = [] + if hasattr(node, 'annotations') and node.annotations: + for anno in node.annotations: + if hasattr(anno, 'name'): + annotations.append(anno.name) + return annotations + + def type_to_string(type_obj) -> str: + """Convert a javalang type object to string.""" + if not type_obj: + return "void" + if hasattr(type_obj, 'name'): + base_type = type_obj.name + # Handle generic types + if hasattr(type_obj, 'arguments') and type_obj.arguments: + args = [type_to_string(arg.type) if hasattr(arg, 'type') else str(arg) + for arg in type_obj.arguments] + return f"{base_type}<{', '.join(args)}>" + # Handle array dimensions + if hasattr(type_obj, 'dimensions') and type_obj.dimensions: + return base_type + "[]" * len(type_obj.dimensions) + return base_type + return str(type_obj) + + # Extract package and imports once + package_name = tree.package.name if hasattr(tree, 'package') and tree.package else None + imports = extract_imports(tree) + + decl_count = 0 for path, node in tree: + metadata = None + chunk_text = None + start_line = 0 + end_line = 0 + + # CLASS DECLARATION if isinstance(node, javalang.tree.ClassDeclaration): start_line = node.position.line - 1 if node.position else 0 end_line = find_brace_block_end(start_line) - chunk_text = "".join(lines[start_line:end_line + 1]) - doc = get_context_with_comments(filepath, start_line + 1) - extends_class = None + chunk_text = "".join(lines[start_line:end_line]) + + extends_class = node.extends.name if node.extends else None implements_interfaces = [] - - if node.extends: - extends_class = node.extends.name if hasattr(node, 'implements') and node.implements: - implements_interfaces = [impl.name for impl in node.implements] + implements_interfaces = [type_to_string(impl) for impl in node.implements] - chunks.append({ - "text": f"File: {filepath}\nType: Class\nName: {node.name}\nExtends: {extends_class or 'None'}\nImplements: {', '.join(implements_interfaces) or 'None'}\nDoc:\n{doc}\nCode:\n{chunk_text}", - "metadata": { - "file": str(filepath), - "name": node.name, - "type": "class", - "line": start_line + 1, - "language": "java", - "extends": extends_class, - "implements": implements_interfaces - } - }) + metadata = { + "file": str(filepath), + "name": node.name, + "type": "class", + "line": start_line + 1, + "language": "java", + "visibility": get_visibility(node.modifiers), + "package": package_name, + "imports": imports, + "extends": extends_class, + "implements": implements_interfaces, + "annotations": extract_annotations(node), + } + # Extract generics + if hasattr(node, 'type_parameters') and node.type_parameters: + metadata["generics"] = [tp.name for tp in node.type_parameters] + + # Check for abstract/final/static modifiers + if node.modifiers: + if 'abstract' in node.modifiers: + metadata["is_abstract"] = True + if 'final' in node.modifiers: + metadata["is_final"] = True + if 'static' in node.modifiers: + metadata["is_static"] = True + + docstring = extract_docstring(node) + if docstring: + metadata["docstring"] = docstring + + # METHOD DECLARATION elif isinstance(node, javalang.tree.MethodDeclaration): if not hasattr(node, 'position') or not node.position: continue + start_line = node.position.line - 1 end_line = find_brace_block_end(start_line) - chunk_text = "".join(lines[start_line:end_line + 1]) - doc = get_context_with_comments(filepath, start_line + 1) - # Find enclosing class name + chunk_text = "".join(lines[start_line:end_line]) + + # Find enclosing class class_name = None for p in reversed(path): if isinstance(p, javalang.tree.ClassDeclaration): class_name = p.name break - return_type = node.return_type.name if node.return_type else "void" + + return_type = type_to_string(node.return_type) parameters = [] if node.parameters: for param in node.parameters: - param_type = param.type.name if param.type else "unknown" - parameters.append(f"{param_type} {param.name}") + param_type = type_to_string(param.type) + param_str = f"{param.name}: {param_type}" + if param.varargs: + param_str = f"{param.name}: {param_type}..." + parameters.append(param_str) - chunks.append({ - "text": f"File: {filepath}\nType: Method\nClass: {class_name or 'unknown'}\nName: {node.name}\nReturn: {return_type}\nParameters: {', '.join(parameters) or 'None'}\nDoc:\n{doc}\nCode:\n{chunk_text}", - "metadata": { - "file": str(filepath), - "name": node.name, - "type": "method", - "class": class_name, - "line": start_line + 1, - "language": "java", - "return_type": return_type, - "parameters": parameters - } - }) + metadata = { + "file": str(filepath), + "name": node.name, + "type": "method", + "class": class_name, + "line": start_line + 1, + "language": "java", + "visibility": get_visibility(node.modifiers), + "package": package_name, + "return_type": return_type, + "parameters": parameters, + "annotations": extract_annotations(node), + } + # Extract generics + if hasattr(node, 'type_parameters') and node.type_parameters: + metadata["generics"] = [tp.name for tp in node.type_parameters] + + # Check modifiers + if node.modifiers: + if 'abstract' in node.modifiers: + metadata["is_abstract"] = True + if 'final' in node.modifiers: + metadata["is_final"] = True + if 'static' in node.modifiers: + metadata["is_static"] = True + if 'synchronized' in node.modifiers: + metadata["is_synchronized"] = True + if 'native' in node.modifiers: + metadata["is_native"] = True + + docstring = extract_docstring(node) + if docstring: + metadata["docstring"] = docstring + + # FIELD DECLARATION elif isinstance(node, javalang.tree.FieldDeclaration): - # Fields may have multiple variable declarators if not hasattr(node, 'position') or not node.position: continue - start_line = node.position.line - 1 - # Field is usually one line, but include next if annotation - end_line = start_line - chunk_text = lines[start_line] - doc = get_context_with_comments(filepath, start_line + 1) - field_type = node.type.name if node.type else "unknown" - for declarator in node.declarators: - chunks.append({ - "text": f"File: {filepath}\nType: Field\nName: {declarator.name}\nField Type: {field_type}\nDoc:\n{doc}\nCode:\n{chunk_text}", - "metadata": { - "file": str(filepath), - "name": declarator.name, - "type": "field", - "line": start_line + 1, - "language": "java", - "field_type": field_type - } - }) + start_line = node.position.line - 1 + end_line = start_line + 1 + chunk_text = lines[start_line] if start_line < len(lines) else "" + + field_type = type_to_string(node.type) + + # Process each declarator (there can be multiple fields on one line) + for declarator in node.declarators: + field_metadata = { + "file": str(filepath), + "name": declarator.name, + "type": "field", + "line": start_line + 1, + "language": "java", + "visibility": get_visibility(node.modifiers), + "package": package_name, + "field_type": field_type, + "annotations": extract_annotations(node), + } + + # Check modifiers + if node.modifiers: + if 'static' in node.modifiers: + field_metadata["is_static"] = True + if 'final' in node.modifiers: + field_metadata["is_final"] = True + if 'volatile' in node.modifiers: + field_metadata["is_volatile"] = True + if 'transient' in node.modifiers: + field_metadata["is_transient"] = True + + docstring = extract_docstring(node) + if docstring: + field_metadata["docstring"] = docstring + + field_text = f"File: {filepath}\nType: Field\nName: {declarator.name}\n" + field_text += f"Field Type: {field_type}\nVisibility: {field_metadata['visibility']}\n" + if field_metadata.get("annotations"): + field_text += f"Annotations: {', '.join(field_metadata['annotations'])}\n" + field_text += f"Code:\n{chunk_text}" + + chunks.append({ + "text": field_text, + "metadata": field_metadata + }) + decl_count += 1 + + # Skip the standard processing since we handled fields inline + continue + + # CONSTRUCTOR DECLARATION elif isinstance(node, javalang.tree.ConstructorDeclaration): if not hasattr(node, 'position') or not node.position: continue + start_line = node.position.line - 1 end_line = find_brace_block_end(start_line) - chunk_text = "".join(lines[start_line:end_line + 1]) - doc = get_context_with_comments(filepath, start_line + 1) + chunk_text = "".join(lines[start_line:end_line]) + + # Find enclosing class class_name = None for p in reversed(path): if isinstance(p, javalang.tree.ClassDeclaration): class_name = p.name break - chunks.append({ - "text": f"File: {filepath}\nType: Constructor\nClass: {class_name or 'unknown'}\nDoc:\n{doc}\nCode:\n{chunk_text}", - "metadata": { - "file": str(filepath), - "name": f"{class_name} constructor", - "type": "constructor", - "class": class_name, - "line": start_line + 1, - "language": "java" - } - }) + parameters = [] + if node.parameters: + for param in node.parameters: + param_type = type_to_string(param.type) + parameters.append(f"{param.name}: {param_type}") + + metadata = { + "file": str(filepath), + "name": f"{class_name}", + "type": "constructor", + "class": class_name, + "line": start_line + 1, + "language": "java", + "visibility": get_visibility(node.modifiers), + "package": package_name, + "parameters": parameters, + "annotations": extract_annotations(node), + } + + docstring = extract_docstring(node) + if docstring: + metadata["docstring"] = docstring + + # ENUM DECLARATION elif isinstance(node, javalang.tree.EnumDeclaration): start_line = node.position.line - 1 if node.position else 0 end_line = find_brace_block_end(start_line) - chunk_text = "".join(lines[start_line:end_line + 1]) - doc = get_context_with_comments(filepath, start_line + 1) - chunks.append({ - "text": f"File: {filepath}\nType: Enum\nName: {node.name}\nDoc:\n{doc}\nCode:\n{chunk_text}", - "metadata": { - "file": str(filepath), - "name": node.name, - "type": "enum", - "line": start_line + 1, - "language": "java" - } - }) + chunk_text = "".join(lines[start_line:end_line]) + # Extract enum constants + variants = [] + if hasattr(node, 'body') and node.body: + for item in node.body: + if isinstance(item, javalang.tree.EnumConstantDeclaration): + variants.append(item.name) + + implements_interfaces = [] + if hasattr(node, 'implements') and node.implements: + implements_interfaces = [type_to_string(impl) for impl in node.implements] + + metadata = { + "file": str(filepath), + "name": node.name, + "type": "enum", + "line": start_line + 1, + "language": "java", + "visibility": get_visibility(node.modifiers), + "package": package_name, + "implements": implements_interfaces, + "variants": variants, + "annotations": extract_annotations(node), + } + + docstring = extract_docstring(node) + if docstring: + metadata["docstring"] = docstring + + # INTERFACE DECLARATION elif isinstance(node, javalang.tree.InterfaceDeclaration): start_line = node.position.line - 1 if node.position else 0 end_line = find_brace_block_end(start_line) - chunk_text = "".join(lines[start_line:end_line + 1]) - doc = get_context_with_comments(filepath, start_line + 1) - chunks.append({ - "text": f"File: {filepath}\nType: Interface\nName: {node.name}\nDoc:\n{doc}\nCode:\n{chunk_text}", - "metadata": { - "file": str(filepath), - "name": node.name, - "type": "interface", - "line": start_line + 1, - "language": "java" - } - }) + chunk_text = "".join(lines[start_line:end_line]) + extends_interfaces = [] + if hasattr(node, 'extends') and node.extends: + extends_interfaces = [type_to_string(ext) for ext in node.extends] + + # Extract method signatures + methods = [] + if hasattr(node, 'body') and node.body: + for item in node.body: + if isinstance(item, javalang.tree.MethodDeclaration): + methods.append(item.name) + + metadata = { + "file": str(filepath), + "name": node.name, + "type": "interface", + "line": start_line + 1, + "language": "java", + "visibility": get_visibility(node.modifiers), + "package": package_name, + "extends": extends_interfaces, + "methods": methods, + "annotations": extract_annotations(node), + } + + # Extract generics + if hasattr(node, 'type_parameters') and node.type_parameters: + metadata["generics"] = [tp.name for tp in node.type_parameters] + + docstring = extract_docstring(node) + if docstring: + metadata["docstring"] = docstring + + # ANNOTATION DECLARATION elif isinstance(node, javalang.tree.AnnotationDeclaration): start_line = node.position.line - 1 if node.position else 0 end_line = find_brace_block_end(start_line) - chunk_text = "".join(lines[start_line:end_line + 1]) - doc = get_context_with_comments(filepath, start_line + 1) + chunk_text = "".join(lines[start_line:end_line]) + + metadata = { + "file": str(filepath), + "name": node.name, + "type": "annotation", + "line": start_line + 1, + "language": "java", + "visibility": get_visibility(node.modifiers), + "package": package_name, + } + + docstring = extract_docstring(node) + if docstring: + metadata["docstring"] = docstring + + # Build chunk text and add to chunks list + if metadata and chunk_text is not None: + text_parts = [f"File: {filepath}"] + text_parts.append(f"Type: {metadata['type']}") + text_parts.append(f"Name: {metadata['name']}") + + if metadata.get('visibility'): + text_parts.append(f"Visibility: {metadata['visibility']}") + if metadata.get('package'): + text_parts.append(f"Package: {metadata['package']}") + if metadata.get('extends'): + text_parts.append(f"Extends: {metadata['extends']}") + if metadata.get('implements'): + text_parts.append(f"Implements: {', '.join(metadata['implements'])}") + if metadata.get('return_type'): + text_parts.append(f"Returns: {metadata['return_type']}") + if metadata.get('parameters'): + text_parts.append(f"Parameters: {', '.join(metadata['parameters'])}") + if metadata.get('annotations'): + text_parts.append(f"Annotations: {', '.join(metadata['annotations'])}") + if metadata.get('generics'): + text_parts.append(f"Generics: {', '.join(metadata['generics'])}") + + text_parts.append(f"Code:\n{chunk_text}") + chunks.append({ - "text": f"File: {filepath}\nType: Annotation\nName: {node.name}\nDoc:\n{doc}\nCode:\n{chunk_text}", - "metadata": { - "file": str(filepath), - "name": node.name, - "type": "annotation", - "line": start_line + 1, - "language": "java" - } + "text": "\n".join(text_parts), + "metadata": metadata }) + decl_count += 1 + + logger.info(f" Parsed {decl_count} declarations from {filepath}") - except javalang.parser.JavaSyntaxError as e: - logger.warning(f"Failed to parse Java file {filepath}: {e}") except Exception as e: - logger.warning(f"Unexpected error parsing Java file {filepath}: {e}") - return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) + logger.warning(f"Failed to parse Java file {filepath}: {e}") + + # Return parsed chunks or fallback + if chunks: + logger.info(f"✅ Successfully parsed {len(chunks)} chunks from {filepath}") + return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) + + # Fallback: emit a whole-file chunk + logger.warning(f"❌ No chunks created from Java AST for {filepath}. Emitting fallback whole-file chunk.") + try: + full_code = filepath.read_text(encoding="utf-8") + except Exception: + full_code = "" + + fallback_meta = { + "file": str(filepath), + "name": filepath.name, + "type": "file", + "line": 1, + "language": "java", + } + return ((full_code, tuple(fallback_meta.items())),) def parse_java_file(filepath: Path) -> List[Dict]: """Parse Java source file with caching.""" - cached_result = parse_java_file_cached(str(filepath)) + fileHash = hash(filepath.read_bytes()) + cached_result = parse_java_file_cached(str(filepath), fileHash) return [{"text": text, "metadata": dict(meta)} for text, meta in cached_result] # ----------------------------- diff --git a/tools/parse_go_ast.go b/tools/parse_go_ast.go index e1c5658..3a95a74 100644 --- a/tools/parse_go_ast.go +++ b/tools/parse_go_ast.go @@ -1,4 +1,4 @@ -// # Build from project root: +// Build from project root: // go build -o tools/parse_go_ast tools/parse_go_ast.go package main @@ -9,24 +9,29 @@ import ( "go/ast" "go/parser" "go/token" - "io/ioutil" "os" "strings" ) type GoDecl struct { Name string `json:"name"` - Type string `json:"type"` // "func", "method", "type", "struct", "interface", "var", "package" + Type string `json:"type"` // "func", "method", "struct", "interface", "type", "var", "const" Receiver string `json:"receiver,omitempty"` // e.g., "*User" FullName string `json:"full_name,omitempty"` - Fields []string `json:"fields,omitempty"` // for structs - Methods []string `json:"methods,omitempty"` // for interfaces + Fields []string `json:"fields,omitempty"` // for structs + Methods []string `json:"methods,omitempty"` // for interfaces + Parameters []string `json:"parameters,omitempty"` // for functions/methods + ReturnType string `json:"return_type,omitempty"` DocComment string `json:"doc_comment,omitempty"` + IsExported bool `json:"is_exported,omitempty"` // starts with capital letter StartLine int `json:"start_line"` EndLine int `json:"end_line"` } func astTypeToString(expr ast.Expr) string { + if expr == nil { + return "" + } switch t := expr.(type) { case *ast.Ident: return t.Name @@ -38,6 +43,16 @@ func astTypeToString(expr ast.Expr) string { return "[]" + astTypeToString(t.Elt) case *ast.MapType: return "map[" + astTypeToString(t.Key) + "]" + astTypeToString(t.Value) + case *ast.InterfaceType: + return "interface{}" + case *ast.StructType: + return "struct{}" + case *ast.FuncType: + return "func" + case *ast.ChanType: + return "chan " + astTypeToString(t.Value) + case *ast.Ellipsis: + return "..." + astTypeToString(t.Elt) default: return "unknown" } @@ -48,55 +63,96 @@ func extractDocComment(comments []*ast.CommentGroup, pos token.Pos, fset *token. return "" } line := fset.Position(pos).Line + + // Look for comments immediately before the declaration for i := len(comments) - 1; i >= 0; i-- { cg := comments[i] - cgLine := fset.Position(cg.End()).Line - if cgLine < line && line-cgLine <= 5 { - return strings.TrimSpace(cg.Text()) + cgEndLine := fset.Position(cg.End()).Line + cgStartLine := fset.Position(cg.Pos()).Line + + // Comment should be within 1 line of the declaration + if cgEndLine < line && line-cgEndLine <= 1 { + text := strings.TrimSpace(cg.Text()) + // Clean up comment markers + text = strings.ReplaceAll(text, "/*", "") + text = strings.ReplaceAll(text, "*/", "") + return strings.TrimSpace(text) + } + + // If we've gone too far back, stop looking + if cgStartLine < line-10 { + break } } return "" } +func extractParameters(fields *ast.FieldList) []string { + if fields == nil { + return nil + } + + var params []string + for _, field := range fields.List { + typeStr := astTypeToString(field.Type) + if len(field.Names) == 0 { + // Unnamed parameter + params = append(params, typeStr) + } else { + for _, name := range field.Names { + params = append(params, name.Name+": "+typeStr) + } + } + } + return params +} + +func extractReturnType(results *ast.FieldList) string { + if results == nil || len(results.List) == 0 { + return "" + } + + if len(results.List) == 1 { + return astTypeToString(results.List[0].Type) + } + + // Multiple return values + var types []string + for _, field := range results.List { + types = append(types, astTypeToString(field.Type)) + } + return "(" + strings.Join(types, ", ") + ")" +} + +func isExported(name string) bool { + if len(name) == 0 { + return false + } + firstRune := []rune(name)[0] + return firstRune >= 'A' && firstRune <= 'Z' +} + func main() { if len(os.Args) < 2 { - fmt.Println("Usage: parse_go_ast ") + fmt.Fprintln(os.Stderr, "Usage: parse_go_ast ") os.Exit(1) } + filename := os.Args[1] - src, err := ioutil.ReadFile(filename) + src, err := os.ReadFile(filename) if err != nil { - fmt.Printf("ERR: %v\n", err) + fmt.Fprintf(os.Stderr, "ERR: Failed to read file: %v\n", err) os.Exit(2) } fset := token.NewFileSet() fileNode, err := parser.ParseFile(fset, filename, src, parser.ParseComments) if err != nil { - fmt.Printf("ERR: %v\n", err) + fmt.Fprintf(os.Stderr, "ERR: Failed to parse Go file: %v\n", err) os.Exit(3) } - decls := []GoDecl{} - - // Package comment (first comment group before package decl) - pkgComment := "" - for _, cg := range fileNode.Comments { - if cg.Pos() < fileNode.Package { - pkgComment = strings.TrimSpace(cg.Text()) - } else { - break - } - } - if pkgComment != "" { - decls = append(decls, GoDecl{ - Name: "package", - Type: "package", - DocComment: pkgComment, - StartLine: 1, - EndLine: 1, - }) - } + var decls []GoDecl // Process declarations for _, d := range fileNode.Decls { @@ -104,25 +160,36 @@ func main() { case *ast.FuncDecl: name := d.Name.Name recvType := "" + if d.Recv != nil && len(d.Recv.List) > 0 { recvType = astTypeToString(d.Recv.List[0].Type) } - typ := "func" + + typ := "function" fullName := name if recvType != "" { typ = "method" fullName = fmt.Sprintf("(%s).%s", recvType, name) } + doc := extractDocComment(fileNode.Comments, d.Pos(), fset) - decls = append(decls, GoDecl{ + params := extractParameters(d.Type.Params) + returnType := extractReturnType(d.Type.Results) + + decl := GoDecl{ Name: name, Type: typ, Receiver: recvType, FullName: fullName, + Parameters: params, + ReturnType: returnType, DocComment: doc, + IsExported: isExported(name), StartLine: fset.Position(d.Pos()).Line, EndLine: fset.Position(d.End()).Line, - }) + } + + decls = append(decls, decl) case *ast.GenDecl: for _, spec := range d.Specs { @@ -136,59 +203,130 @@ func main() { switch t := s.Type.(type) { case *ast.StructType: declType = "struct" - for _, f := range t.Fields.List { - for _, n := range f.Names { - field := n.Name - if f.Tag != nil { - field += " " + f.Tag.Value + if t.Fields != nil { + for _, f := range t.Fields.List { + fieldType := astTypeToString(f.Type) + + if len(f.Names) == 0 { + // Embedded field + fields = append(fields, "embedded:"+fieldType) + } else { + // Named fields + for _, n := range f.Names { + field := n.Name + ": " + fieldType + if f.Tag != nil { + field += " " + f.Tag.Value + } + fields = append(fields, field) + } } - fields = append(fields, field) - } - if f.Names == nil { - // Embedded field - fields = append(fields, astTypeToString(f.Type)) } } + case *ast.InterfaceType: declType = "interface" - for _, m := range t.Methods.List { - if len(m.Names) > 0 { - methods = append(methods, m.Names[0].Name) + if t.Methods != nil { + for _, m := range t.Methods.List { + if len(m.Names) > 0 { + // Named method + for _, name := range m.Names { + methods = append(methods, name.Name) + } + } else { + // Embedded interface + embeddedType := astTypeToString(m.Type) + methods = append(methods, "embedded:"+embeddedType) + } } } } doc := extractDocComment(fileNode.Comments, s.Pos(), fset) - decls = append(decls, GoDecl{ + decl := GoDecl{ Name: typName, Type: declType, Fields: fields, Methods: methods, DocComment: doc, + IsExported: isExported(typName), StartLine: fset.Position(d.Pos()).Line, EndLine: fset.Position(d.End()).Line, - }) + } + + decls = append(decls, decl) case *ast.ValueSpec: + // Variable or constant declarations + varType := "var" + if d.Tok == token.CONST { + varType = "const" + } + for _, name := range s.Names { doc := extractDocComment(fileNode.Comments, name.Pos(), fset) - // Use the individual identifier's position, not the declaration's position + + // Get the type if specified + typeStr := "" + if s.Type != nil { + typeStr = astTypeToString(s.Type) + } + + // Use the individual identifier's position startLine := fset.Position(name.Pos()).Line endLine := fset.Position(name.End()).Line - - decls = append(decls, GoDecl{ + + decl := GoDecl{ Name: name.Name, - Type: "var", + Type: varType, + ReturnType: typeStr, // Reuse return_type field for variable type DocComment: doc, + IsExported: isExported(name.Name), StartLine: startLine, EndLine: endLine, - }) + } + + decls = append(decls, decl) + } + + case *ast.ImportSpec: + // Track imports for graph building + importPath := "" + if s.Path != nil { + importPath = strings.Trim(s.Path.Value, "\"") + } + + importName := "" + if s.Name != nil { + importName = s.Name.Name + } else { + // Extract package name from path + parts := strings.Split(importPath, "/") + if len(parts) > 0 { + importName = parts[len(parts)-1] + } + } + + if importPath != "" { + decl := GoDecl{ + Name: importName, + Type: "import", + FullName: importPath, + StartLine: fset.Position(s.Pos()).Line, + EndLine: fset.Position(s.End()).Line, + } + decls = append(decls, decl) } } } } } - out, _ := json.Marshal(decls) + // Marshal to JSON and output + out, err := json.Marshal(decls) + if err != nil { + fmt.Fprintf(os.Stderr, "ERR: Failed to marshal JSON: %v\n", err) + os.Exit(4) + } + fmt.Println(string(out)) }