JAVA and GOLANG parser update

tools/parse_go_ast.go

@@ -1,4 +1,4 @@
-// # Build from project root:
+// Build from project root:
 // go build -o tools/parse_go_ast tools/parse_go_ast.go
 
 package main
@@ -9,24 +9,29 @@ import (
 	"go/ast"
 	"go/parser"
 	"go/token"
-	"io/ioutil"
 	"os"
 	"strings"
 )
 
 type GoDecl struct {
 	Name       string   `json:"name"`
-	Type       string   `json:"type"`               // "func", "method", "type", "struct", "interface", "var", "package"
+	Type       string   `json:"type"`               // "func", "method", "struct", "interface", "type", "var", "const"
 	Receiver   string   `json:"receiver,omitempty"` // e.g., "*User"
 	FullName   string   `json:"full_name,omitempty"`
-	Fields     []string `json:"fields,omitempty"`  // for structs
+	Fields     []string `json:"fields,omitempty"`     // for structs
-	Methods    []string `json:"methods,omitempty"` // for interfaces
+	Methods    []string `json:"methods,omitempty"`    // for interfaces
+	Parameters []string `json:"parameters,omitempty"` // for functions/methods
+	ReturnType string   `json:"return_type,omitempty"`
 	DocComment string   `json:"doc_comment,omitempty"`
+	IsExported bool     `json:"is_exported,omitempty"` // starts with capital letter
 	StartLine  int      `json:"start_line"`
 	EndLine    int      `json:"end_line"`
 }
 
 func astTypeToString(expr ast.Expr) string {
+	if expr == nil {
+		return ""
+	}
 	switch t := expr.(type) {
 	case *ast.Ident:
 		return t.Name
@@ -38,6 +43,16 @@ func astTypeToString(expr ast.Expr) string {
 		return "[]" + astTypeToString(t.Elt)
 	case *ast.MapType:
 		return "map[" + astTypeToString(t.Key) + "]" + astTypeToString(t.Value)
+	case *ast.InterfaceType:
+		return "interface{}"
+	case *ast.StructType:
+		return "struct{}"
+	case *ast.FuncType:
+		return "func"
+	case *ast.ChanType:
+		return "chan " + astTypeToString(t.Value)
+	case *ast.Ellipsis:
+		return "..." + astTypeToString(t.Elt)
 	default:
 		return "unknown"
 	}
@@ -48,55 +63,96 @@ func extractDocComment(comments []*ast.CommentGroup, pos token.Pos, fset *token.
 		return ""
 	}
 	line := fset.Position(pos).Line
+
+	// Look for comments immediately before the declaration
 	for i := len(comments) - 1; i >= 0; i-- {
 		cg := comments[i]
-		cgLine := fset.Position(cg.End()).Line
+		cgEndLine := fset.Position(cg.End()).Line
-		if cgLine < line && line-cgLine <= 5 {
+		cgStartLine := fset.Position(cg.Pos()).Line
-			return strings.TrimSpace(cg.Text())
+
+		// Comment should be within 1 line of the declaration
+		if cgEndLine < line && line-cgEndLine <= 1 {
+			text := strings.TrimSpace(cg.Text())
+			// Clean up comment markers
+			text = strings.ReplaceAll(text, "/*", "")
+			text = strings.ReplaceAll(text, "*/", "")
+			return strings.TrimSpace(text)
+		}
+
+		// If we've gone too far back, stop looking
+		if cgStartLine < line-10 {
+			break
 		}
 	}
 	return ""
 }
 
+func extractParameters(fields *ast.FieldList) []string {
+	if fields == nil {
+		return nil
+	}
+
+	var params []string
+	for _, field := range fields.List {
+		typeStr := astTypeToString(field.Type)
+		if len(field.Names) == 0 {
+			// Unnamed parameter
+			params = append(params, typeStr)
+		} else {
+			for _, name := range field.Names {
+				params = append(params, name.Name+": "+typeStr)
+			}
+		}
+	}
+	return params
+}
+
+func extractReturnType(results *ast.FieldList) string {
+	if results == nil || len(results.List) == 0 {
+		return ""
+	}
+
+	if len(results.List) == 1 {
+		return astTypeToString(results.List[0].Type)
+	}
+
+	// Multiple return values
+	var types []string
+	for _, field := range results.List {
+		types = append(types, astTypeToString(field.Type))
+	}
+	return "(" + strings.Join(types, ", ") + ")"
+}
+
+func isExported(name string) bool {
+	if len(name) == 0 {
+		return false
+	}
+	firstRune := []rune(name)[0]
+	return firstRune >= 'A' && firstRune <= 'Z'
+}
+
 func main() {
 	if len(os.Args) < 2 {
-		fmt.Println("Usage: parse_go_ast <file.go>")
+		fmt.Fprintln(os.Stderr, "Usage: parse_go_ast <file.go>")
 		os.Exit(1)
 	}
+
 	filename := os.Args[1]
-	src, err := ioutil.ReadFile(filename)
+	src, err := os.ReadFile(filename)
 	if err != nil {
-		fmt.Printf("ERR: %v\n", err)
+		fmt.Fprintf(os.Stderr, "ERR: Failed to read file: %v\n", err)
 		os.Exit(2)
 	}
 
 	fset := token.NewFileSet()
 	fileNode, err := parser.ParseFile(fset, filename, src, parser.ParseComments)
 	if err != nil {
-		fmt.Printf("ERR: %v\n", err)
+		fmt.Fprintf(os.Stderr, "ERR: Failed to parse Go file: %v\n", err)
 		os.Exit(3)
 	}
 
-	decls := []GoDecl{}
+	var decls []GoDecl
-
-	// Package comment (first comment group before package decl)
-	pkgComment := ""
-	for _, cg := range fileNode.Comments {
-		if cg.Pos() < fileNode.Package {
-			pkgComment = strings.TrimSpace(cg.Text())
-		} else {
-			break
-		}
-	}
-	if pkgComment != "" {
-		decls = append(decls, GoDecl{
-			Name:       "package",
-			Type:       "package",
-			DocComment: pkgComment,
-			StartLine:  1,
-			EndLine:    1,
-		})
-	}
 
 	// Process declarations
 	for _, d := range fileNode.Decls {
@@ -104,25 +160,36 @@ func main() {
 		case *ast.FuncDecl:
 			name := d.Name.Name
 			recvType := ""
+
 			if d.Recv != nil && len(d.Recv.List) > 0 {
 				recvType = astTypeToString(d.Recv.List[0].Type)
 			}
-			typ := "func"
+
+			typ := "function"
 			fullName := name
 			if recvType != "" {
 				typ = "method"
 				fullName = fmt.Sprintf("(%s).%s", recvType, name)
 			}
+
 			doc := extractDocComment(fileNode.Comments, d.Pos(), fset)
-			decls = append(decls, GoDecl{
+			params := extractParameters(d.Type.Params)
+			returnType := extractReturnType(d.Type.Results)
+
+			decl := GoDecl{
 				Name:       name,
 				Type:       typ,
 				Receiver:   recvType,
 				FullName:   fullName,
+				Parameters: params,
+				ReturnType: returnType,
 				DocComment: doc,
+				IsExported: isExported(name),
 				StartLine:  fset.Position(d.Pos()).Line,
 				EndLine:    fset.Position(d.End()).Line,
-			})
+			}
+
+			decls = append(decls, decl)
 
 		case *ast.GenDecl:
 			for _, spec := range d.Specs {
@@ -136,59 +203,130 @@ func main() {
 					switch t := s.Type.(type) {
 					case *ast.StructType:
 						declType = "struct"
-						for _, f := range t.Fields.List {
+						if t.Fields != nil {
-							for _, n := range f.Names {
+							for _, f := range t.Fields.List {
-								field := n.Name
+								fieldType := astTypeToString(f.Type)
-								if f.Tag != nil {
+
-									field += " " + f.Tag.Value
+								if len(f.Names) == 0 {
+									// Embedded field
+									fields = append(fields, "embedded:"+fieldType)
+								} else {
+									// Named fields
+									for _, n := range f.Names {
+										field := n.Name + ": " + fieldType
+										if f.Tag != nil {
+											field += " " + f.Tag.Value
+										}
+										fields = append(fields, field)
+									}
 								}
-								fields = append(fields, field)
-							}
-							if f.Names == nil {
-								// Embedded field
-								fields = append(fields, astTypeToString(f.Type))
 							}
 						}
+
 					case *ast.InterfaceType:
 						declType = "interface"
-						for _, m := range t.Methods.List {
+						if t.Methods != nil {
-							if len(m.Names) > 0 {
+							for _, m := range t.Methods.List {
-								methods = append(methods, m.Names[0].Name)
+								if len(m.Names) > 0 {
+									// Named method
+									for _, name := range m.Names {
+										methods = append(methods, name.Name)
+									}
+								} else {
+									// Embedded interface
+									embeddedType := astTypeToString(m.Type)
+									methods = append(methods, "embedded:"+embeddedType)
+								}
 							}
 						}
 					}
 
 					doc := extractDocComment(fileNode.Comments, s.Pos(), fset)
-					decls = append(decls, GoDecl{
+					decl := GoDecl{
 						Name:       typName,
 						Type:       declType,
 						Fields:     fields,
 						Methods:    methods,
 						DocComment: doc,
+						IsExported: isExported(typName),
 						StartLine:  fset.Position(d.Pos()).Line,
 						EndLine:    fset.Position(d.End()).Line,
-					})
+					}
+
+					decls = append(decls, decl)
 
 				case *ast.ValueSpec:
+					// Variable or constant declarations
+					varType := "var"
+					if d.Tok == token.CONST {
+						varType = "const"
+					}
+
 					for _, name := range s.Names {
 						doc := extractDocComment(fileNode.Comments, name.Pos(), fset)
-						// Use the individual identifier's position, not the declaration's position
+
+						// Get the type if specified
+						typeStr := ""
+						if s.Type != nil {
+							typeStr = astTypeToString(s.Type)
+						}
+
+						// Use the individual identifier's position
 						startLine := fset.Position(name.Pos()).Line
 						endLine := fset.Position(name.End()).Line
 
-						decls = append(decls, GoDecl{
+						decl := GoDecl{
 							Name:       name.Name,
-							Type:       "var",
+							Type:       varType,
+							ReturnType: typeStr, // Reuse return_type field for variable type
 							DocComment: doc,
+							IsExported: isExported(name.Name),
 							StartLine:  startLine,
 							EndLine:    endLine,
-						})
+						}
+
+						decls = append(decls, decl)
+					}
+
+				case *ast.ImportSpec:
+					// Track imports for graph building
+					importPath := ""
+					if s.Path != nil {
+						importPath = strings.Trim(s.Path.Value, "\"")
+					}
+
+					importName := ""
+					if s.Name != nil {
+						importName = s.Name.Name
+					} else {
+						// Extract package name from path
+						parts := strings.Split(importPath, "/")
+						if len(parts) > 0 {
+							importName = parts[len(parts)-1]
+						}
+					}
+
+					if importPath != "" {
+						decl := GoDecl{
+							Name:      importName,
+							Type:      "import",
+							FullName:  importPath,
+							StartLine: fset.Position(s.Pos()).Line,
+							EndLine:   fset.Position(s.End()).Line,
+						}
+						decls = append(decls, decl)
 					}
 				}
 			}
 		}
 	}
 
-	out, _ := json.Marshal(decls)
+	// Marshal to JSON and output
+	out, err := json.Marshal(decls)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "ERR: Failed to marshal JSON: %v\n", err)
+		os.Exit(4)
+	}
+
 	fmt.Println(string(out))
 }