popertots/ragmcp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better rust parser

Browse Source
This commit is contained in:
popertots
2025-11-16 00:07:23 +00:00
parent 76bab8abc2
commit 862c372310
2 changed files with 350 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/src/main.rs
+284 -84
View File
@@ -7,7 +7,7 @@ use std::fs;
use std::process;
use syn::{
spanned::Spanned, Attribute, FnArg, Item, ItemEnum, ItemFn, ItemImpl, ItemMod, ItemStruct,
ItemTrait, Pat, Type, Visibility,
ItemTrait, Pat, Type, UseTree, UseTreeKind, Visibility,
};
#[derive(Serialize)]
@@ -26,6 +26,8 @@ struct RustDecl {
return_type: Option<String>,
parameters: Vec<String>,
docstring: Option<String>,
imports: Vec<String>, // Track imports for cross-file relationships
calls: Vec<String>, // Track function calls
}
fn main() {
@@ -53,37 +55,73 @@ fn main() {
};
let mut decls = Vec::new();
let mut imports = Vec::new(); // Collect imports separately
// Process declarations
// Process imports first
for item in &syntax.items {
if let Item::Use(item_use) = item {
let import_paths = extract_use_paths(&item_use.tree);
imports.extend(import_paths.clone());
for path in &import_paths {
decls.push(RustDecl {
name: path.clone(),
type_: "import".to_string(),
start_line: get_span_start(&item_use.span()),
end_line: get_span_end(&item_use.span()),
visibility: visibility_to_string(&item_use.vis),
is_async: false,
is_unsafe: false,
generics: vec![],
traits: vec![],
fields: vec![],
methods: vec![],
return_type: None,
parameters: vec![path.clone()],
docstring: extract_docstring(&item_use.attrs),
imports: vec![],
calls: vec![],
});
}
}
}
// Process other declarations with import context
for item in &syntax.items {
match item {
Item::Fn(item_fn) => {
if let Some(decl) = parse_function(&item_fn) {
if let Some(mut decl) = parse_function(&item_fn, &content) {
decl.imports = imports.clone(); // Add imports to function context
decls.push(decl);
}
}
Item::Struct(item_struct) => {
if let Some(decl) = parse_struct(&item_struct) {
if let Some(mut decl) = parse_struct(&item_struct) {
decl.imports = imports.clone();
decls.push(decl);
}
}
Item::Enum(item_enum) => {
if let Some(decl) = parse_enum(&item_enum) {
if let Some(mut decl) = parse_enum(&item_enum) {
decl.imports = imports.clone();
decls.push(decl);
}
}
Item::Trait(item_trait) => {
if let Some(decl) = parse_trait(&item_trait) {
if let Some(mut decl) = parse_trait(&item_trait) {
decl.imports = imports.clone();
decls.push(decl);
}
}
Item::Impl(item_impl) => {
if let Some(decl) = parse_impl(&item_impl) {
if let Some(mut decl) = parse_impl(&item_impl, &content) {
decl.imports = imports.clone();
decls.push(decl);
}
}
Item::Mod(item_mod) => {
if let Some(decl) = parse_mod(&item_mod) {
if let Some(mut decl) = parse_mod(&item_mod) {
decl.imports = imports.clone();
decls.push(decl);
}
}
@@ -91,43 +129,74 @@ fn main() {
}
}
for item in &syntax.items {
if let Item::Use(item_use) = item {
let path = use_tree_to_string(&item_use.tree);
decls.push(RustDecl {
name: path.clone(),
type_: "import".to_string(),
start_line: 0,
end_line: 0,
visibility: "".to_string(),
is_async: false,
is_unsafe: false,
generics: vec![],
traits: vec![],
fields: vec![],
methods: vec![],
return_type: None,
parameters: vec![path],
docstring: None,
});
}
}
let output = serde_json::to_string(&decls).unwrap_or_else(|_| "[]".to_string());
println!("{}", output);
}
fn span_start_end<T: Spanned>(node: &T) -> (usize, usize) {
// Requires proc-macro2 "span-locations" feature.
let span = node.span();
let start = span.start().line;
let end = span.end().line;
(start, end)
// Better span handling (requires proc-macro2 "span-locations" feature)
fn get_span_start(span: &proc_macro2::Span) -> usize {
span.start().line
}
fn parse_function(item_fn: &ItemFn) -> Option<RustDecl> {
let (start_line, end_line) = span_start_end(item_fn);
fn get_span_end(span: &proc_macro2::Span) -> usize {
span.end().line
}
// Enhanced import extraction
fn extract_use_paths(tree: &UseTree) -> Vec<String> {
let mut paths = Vec::new();
extract_use_paths_recursive(tree, "", &mut paths);
paths
}
fn extract_use_paths_recursive(tree: &UseTree, current_path: &str, paths: &mut Vec<String>) {
match tree {
UseTree::Path(path) => {
let new_path = if current_path.is_empty() {
path.ident.to_string()
} else {
format!("{}::{}", current_path, path.ident)
};
extract_use_paths_recursive(&path.tree, &new_path, paths);
}
UseTree::Name(name) => {
let full_path = if current_path.is_empty() {
name.ident.to_string()
} else {
format!("{}::{}", current_path, name.ident)
};
paths.push(full_path);
}
UseTree::Rename(rename) => {
let full_path = if current_path.is_empty() {
rename.ident.to_string()
} else {
format!("{}::{}", current_path, rename.ident)
};
paths.push(full_path);
}
UseTree::Glob(_) => {
if !current_path.is_empty() {
paths.push(format!("{}::*", current_path));
} else {
// top-level glob — represent as "*"
paths.push("*".to_string());
}
}
UseTree::Group(group) => {
for item in &group.items {
extract_use_paths_recursive(item, current_path, paths);
}
}
// If the AST contains other nodes, ignore them (keeps function exhaustive)
_ => {}
}
}
// Enhanced function parsing with call detection
fn parse_function(item_fn: &ItemFn, file_content: &str) -> Option<RustDecl> {
let start_line = get_span_start(&item_fn.span());
let end_line = get_span_end(&item_fn.span());
let mut parameters = Vec::new();
for input in &item_fn.sig.inputs {
@@ -149,6 +218,9 @@ fn parse_function(item_fn: &ItemFn) -> Option<RustDecl> {
syn::ReturnType::Type(_, ty) => Some(type_to_string(&*ty)),
};
// Extract function calls from body
let calls = extract_function_calls(&item_fn.block, file_content);
Some(RustDecl {
name: item_fn.sig.ident.to_string(),
type_: "function".to_string(),
@@ -170,16 +242,100 @@ fn parse_function(item_fn: &ItemFn) -> Option<RustDecl> {
return_type,
parameters,
docstring: extract_docstring(&item_fn.attrs),
imports: Vec::new(), // Filled by caller
calls,
})
}
// Simple function call extraction using regex on tokenized block
fn extract_function_calls(block: &syn::Block, _file_content: &str) -> Vec<String> {
let mut calls = Vec::new();
let content = quote::ToTokens::to_token_stream(block).to_string();
// Basic regex to find function-like identifiers followed by '('
let re = Regex::new(r"\b([a-zA-Z_][a-zA-Z0-9_]*)\s*\(").unwrap();
for cap in re.captures_iter(&content) {
if let Some(name) = cap.get(1) {
let call = name.as_str().to_string();
// Filter out common keywords and types, and constructors (capitalized)
if !is_rust_keyword(&call) && !call.chars().next().map_or(false, |c| c.is_uppercase()) {
calls.push(call);
}
}
}
// Deduplicate while preserving order
let mut deduped = Vec::new();
for c in calls {
if !deduped.contains(&c) {
deduped.push(c);
}
}
deduped
}
fn is_rust_keyword(word: &str) -> bool {
matches!(
word,
"if" | "else"
| "for"
| "while"
| "loop"
| "match"
| "return"
| "break"
| "continue"
| "let"
| "mut"
| "const"
| "static"
| "fn"
| "struct"
| "enum"
| "trait"
| "impl"
| "mod"
| "use"
| "pub"
| "crate"
| "self"
| "super"
| "async"
| "await"
| "unsafe"
| "dyn"
| "move"
| "where"
| "in"
| "type"
| "as"
| "ref"
| "box"
| "true"
| "false"
| "Some"
| "None"
| "Ok"
| "Err"
)
}
// Update other parse functions to include the new fields...
fn parse_struct(item_struct: &ItemStruct) -> Option<RustDecl> {
let (start_line, end_line) = span_start_end(item_struct);
let start_line = get_span_start(&item_struct.span());
let end_line = get_span_end(&item_struct.span());
let fields: Vec<String> = item_struct
.fields
.iter()
.filter_map(|field| field.ident.as_ref().map(|ident| ident.to_string()))
.filter_map(|field| {
// field.ident is Option<Ident> for named fields; for tuple structs it's None
field.ident.as_ref().map(|ident| {
let field_name = ident.to_string();
let field_type = type_to_string(&field.ty);
format!("{}: {}", field_name, field_type)
})
})
.collect();
Some(RustDecl {
@@ -202,11 +358,14 @@ fn parse_struct(item_struct: &ItemStruct) -> Option<RustDecl> {
return_type: None,
parameters: Vec::new(),
docstring: extract_docstring(&item_struct.attrs),
imports: Vec::new(), // Filled by caller
calls: Vec::new(),
})
}
fn parse_enum(item_enum: &ItemEnum) -> Option<RustDecl> {
let (start_line, end_line) = span_start_end(item_enum);
let start_line = get_span_start(&item_enum.span());
let end_line = get_span_end(&item_enum.span());
let variants: Vec<String> = item_enum
.variants
@@ -234,11 +393,14 @@ fn parse_enum(item_enum: &ItemEnum) -> Option<RustDecl> {
return_type: None,
parameters: Vec::new(),
docstring: extract_docstring(&item_enum.attrs),
imports: Vec::new(),
calls: Vec::new(),
})
}
fn parse_trait(item_trait: &ItemTrait) -> Option<RustDecl> {
let (start_line, end_line) = span_start_end(item_trait);
let start_line = get_span_start(&item_trait.span());
let end_line = get_span_end(&item_trait.span());
let methods: Vec<String> = item_trait
.items
@@ -272,11 +434,14 @@ fn parse_trait(item_trait: &ItemTrait) -> Option<RustDecl> {
return_type: None,
parameters: Vec::new(),
docstring: extract_docstring(&item_trait.attrs),
imports: Vec::new(),
calls: Vec::new(),
})
}
fn parse_impl(item_impl: &ItemImpl) -> Option<RustDecl> {
let (start_line, end_line) = span_start_end(item_impl);
fn parse_impl(item_impl: &ItemImpl, file_content: &str) -> Option<RustDecl> {
let start_line = get_span_start(&item_impl.span());
let end_line = get_span_end(&item_impl.span());
let target = type_to_string(&*item_impl.self_ty);
@@ -293,21 +458,33 @@ fn parse_impl(item_impl: &ItemImpl) -> Option<RustDecl> {
.collect();
// If this impl implements a trait, extract its path
let implements = item_impl
let implements: Vec<String> = item_impl
.trait_
.as_ref()
.map(|(_, path, _)| path_to_string(path))
.into_iter()
.collect::<Vec<_>>();
.collect();
// Prefer the first trait name (if any) as the identifier; otherwise fall back to the target type
let name = if !implements.is_empty() {
implements[0].clone()
} else {
target
target.clone()
};
// Extract any docstring attached to the impl block
// Extract calls across all method bodies in the impl (basic)
let mut calls = Vec::new();
for item in &item_impl.items {
if let syn::ImplItem::Fn(method) = item {
let body_calls = extract_function_calls(&method.block, file_content);
for c in body_calls {
if !calls.contains(&c) {
calls.push(c);
}
}
}
}
let docstring = extract_docstring(&item_impl.attrs);
Some(RustDecl {
@@ -330,27 +507,14 @@ fn parse_impl(item_impl: &ItemImpl) -> Option<RustDecl> {
return_type: None,
parameters: Vec::new(),
docstring,
imports: Vec::new(),
calls,
})
}
/// Pull a concatenated doc‑string from a list of attributes.
/// Handles both `///` comments (converted to `#[doc = "..."]` by syn)
/// and explicit `#[doc = "..."]` attributes.
fn extract_docstring(attrs: &[Attribute]) -> Option<String> {
// Convert the attribute’s token stream back to source‑like text.
let text = attrs
.iter()
.map(|a| a.to_token_stream().to_string())
.collect::<Vec<_>>()
.join("\n");
let re = Regex::new(r#"(?m)^\s*(?:///|#\[\s*doc\s*=\s*")(.+?)(?:")?$"#).ok()?;
re.captures(&text)
.and_then(|c| c.get(1).map(|m| m.as_str().to_string()))
}
fn parse_mod(item_mod: &ItemMod) -> Option<RustDecl> {
let (start_line, end_line) = span_start_end(item_mod);
let start_line = get_span_start(&item_mod.span());
let end_line = get_span_end(&item_mod.span());
Some(RustDecl {
name: item_mod.ident.to_string(),
@@ -367,6 +531,8 @@ fn parse_mod(item_mod: &ItemMod) -> Option<RustDecl> {
return_type: None,
parameters: Vec::new(),
docstring: extract_docstring(&item_mod.attrs),
imports: Vec::new(),
calls: Vec::new(),
})
}
@@ -375,25 +541,28 @@ fn visibility_to_string(vis: &Visibility) -> String {
Visibility::Public(_) => "pub".to_string(),
Visibility::Restricted(r) => {
// pub(crate) or pub(in ...)
format!("pub({})", quote::ToTokens::to_token_stream(r).to_string())
quote::ToTokens::to_token_stream(r).to_string()
}
_ => "private".to_string(),
}
}
fn type_to_string(ty: &Type) -> String {
// Simple string representation for common types
match ty {
Type::Path(type_path) => type_path
.path
.segments
.last()
.iter()
.map(|seg| seg.ident.to_string())
.unwrap_or_else(|| "unknown".to_string()),
.collect::<Vec<_>>()
.join("::"),
Type::Reference(type_ref) => {
// type_ref.elem is Box<Type>
format!("&{}", type_to_string(&*type_ref.elem))
}
Type::Tuple(tuple) => {
let elems: Vec<String> = tuple.elems.iter().map(type_to_string).collect();
format!("({})", elems.join(", "))
}
_ => quote::ToTokens::to_token_stream(ty).to_string(),
}
}
@@ -414,19 +583,50 @@ fn generic_param_to_string(param: &syn::GenericParam) -> String {
}
}
fn use_tree_to_string(tree: &syn::UseTree) -> String {
match tree {
syn::UseTree::Path(p) => {
let base = p.ident.to_string();
let nested = use_tree_to_string(&*p.tree);
format!("{}::{}", base, nested)
}
syn::UseTree::Name(n) => n.ident.to_string(),
syn::UseTree::Rename(r) => r.ident.to_string(),
syn::UseTree::Glob(_) => "*".to_string(),
syn::UseTree::Group(g) => {
let items: Vec<String> = g.items.iter().map(use_tree_to_string).collect();
items.join("::")
/// Pull a concatenated doc-string from a list of attributes.
/// Handles both `///` comments (converted to `#[doc = "..."]` by syn)
/// and explicit `#[doc = "..."]` attributes.
fn extract_docstring(attrs: &[Attribute]) -> Option<String> {
// Prefer parsing doc attributes explicitly
let mut lines: Vec<String> = Vec::new();
for attr in attrs {
// If attribute path is "doc", try to parse the meta and extract the literal
if attr.path().is_ident("doc") {
if let Ok(syn::Meta::NameValue(nv)) = attr.parse_meta() {
if let syn::Lit::Str(litstr) = nv.lit {
lines.push(litstr.value());
}
} else {
// Fallback: stringify the tokens and try regex extraction
let tok = attr.to_token_stream().to_string();
let re = Regex::new(r#"(?m)#\s*\[\s*doc\s*=\s*"(.+?)"\s*\]"#).ok();
if let Some(re) = re {
if let Some(cap) = re.captures(&tok) {
if let Some(m) = cap.get(1) {
lines.push(m.as_str().to_string());
}
}
}
}
} else {
// capture leading "///" style comments as attributes may not always be "doc"
let tok = attr.to_token_stream().to_string();
// crude fallback to capture /// "..." inside tokens
let re_line = Regex::new(r#"(?m)///\s*(.+)$"#).ok();
if let Some(re_line) = re_line {
for cap in re_line.captures_iter(&tok) {
if let Some(m) = cap.get(1) {
lines.push(m.as_str().to_string());
}
}
}
}
}
if lines.is_empty() {
None
} else {
Some(lines.join("\n"))
}
}