diff --git a/mcp_codebase.py b/mcp_codebase.py index 815cce0..7775b31 100644 --- a/mcp_codebase.py +++ b/mcp_codebase.py @@ -68,19 +68,19 @@ CONTEXTUAL_WEIGHTS = { "sql_function": 1.2, # Function/procedure definitions "sql_view": 1.15, # VIEW definitions "sql_index": 1.1, # INDEX definitions - + # Python weights "python_class": 1.2, # Class definitions "python_function": 1.15, # Function definitions - + # Go weights "go_type": 1.2, # Type definitions "go_function": 1.15, # Function definitions - + # Java weights "java_class": 1.2, # Class definitions "java_method": 1.15, # Method definitions - + # Default weight for unspecified types "default": 1.0 } @@ -137,7 +137,7 @@ def calculate_contextual_weight(metadata: Dict[str, Any]) -> float: """ language = metadata.get("language", "").lower() chunk_type = metadata.get("type", "").lower() - + # SQL-specific weights if language == "sql": if chunk_type in ("create", "alter", "create table", "alter table"): @@ -148,60 +148,60 @@ def calculate_contextual_weight(metadata: Dict[str, Any]) -> float: return CONTEXTUAL_WEIGHTS.get("sql_view", 1.0) elif chunk_type in ("index", "create index"): return CONTEXTUAL_WEIGHTS.get("sql_index", 1.0) - + # Python-specific weights elif language == "python": if chunk_type == "classdef": return CONTEXTUAL_WEIGHTS.get("python_class", 1.0) elif chunk_type in ("functiondef", "asyncfunctiondef"): return CONTEXTUAL_WEIGHTS.get("python_function", 1.0) - + # Go-specific weights elif language == "go": if chunk_type in ("type", "struct", "interface"): return CONTEXTUAL_WEIGHTS.get("go_type", 1.0) elif chunk_type in ("func", "method"): return CONTEXTUAL_WEIGHTS.get("go_function", 1.0) - + # Java-specific weights elif language == "java": if chunk_type == "class": return CONTEXTUAL_WEIGHTS.get("java_class", 1.0) elif chunk_type == "method": return CONTEXTUAL_WEIGHTS.get("java_method", 1.0) - + return CONTEXTUAL_WEIGHTS.get("default", 1.0) -def apply_contextual_weights_to_embeddings(embeddings_list: List[List[float]], +def apply_contextual_weights_to_embeddings(embeddings_list: List[List[float]], metadata_list: List[Dict[str, Any]]) -> List[List[float]]: """ Apply contextual weights to embedding vectors by scaling them. This biases the vector space without changing the embedding model. """ weighted_embeddings = [] - + for embedding, metadata in zip(embeddings_list, metadata_list): weight = calculate_contextual_weight(metadata) - + # Convert to numpy for easier manipulation emb_array = np.array(embedding) - + # Scale the embedding vector by the weight # This effectively increases the magnitude, making it more "important" weighted_emb = emb_array * weight - + # Optionally normalize to maintain consistent vector magnitudes # Comment out if you want the raw weighted vectors norm = np.linalg.norm(weighted_emb) if norm > 0: weighted_emb = weighted_emb / norm * np.linalg.norm(emb_array) - + weighted_embeddings.append(weighted_emb.tolist()) - + if weight != 1.0: logger.debug(f"Applied weight {weight:.2f} to {metadata.get('file')}:{metadata.get('type')}") - + return weighted_embeddings @@ -271,7 +271,7 @@ def parse_sql_file_cached(filepath_str: str, file_hash: str) -> tuple: statements = [s for s in sqlparse.split(sql_code) if s and s.strip()] lines = sql_code.splitlines() search_pos = 0 - + for stmt in statements: start_idx = sql_code.find(stmt, search_pos) if start_idx == -1: @@ -284,7 +284,7 @@ def parse_sql_file_cached(filepath_str: str, file_hash: str) -> tuple: leading_comments = include_leading_comments(lines, start_line) stmt_clean = stmt.strip() - + meta: Dict[str, Any] = { "file": str(filepath), "name": None, @@ -292,7 +292,7 @@ def parse_sql_file_cached(filepath_str: str, file_hash: str) -> tuple: "line": start_line, "language": "sql" } - + try: # Special handling for CREATE TYPE ENUM (PostgreSQL) enum_match = re.match( @@ -306,15 +306,15 @@ def parse_sql_file_cached(filepath_str: str, file_hash: str) -> tuple: meta["type"] = "create type enum" meta["name"] = enum_name meta["enum_values"] = ",".join([v.strip().strip("'\"") for v in enum_values.split(",") if v.strip()]) - + chunk_text = f"File: {filepath}\nType: {meta['type']}\nName: {meta['name']}\n" if leading_comments: chunk_text += f"\nComments:\n{leading_comments}\n\n" chunk_text += f"SQL:\n{stmt_clean}" - + chunks.append({"text": chunk_text, "metadata": meta}) continue - + # Special handling for CREATE EXTENSION extension_match = re.match( r'CREATE\s+EXTENSION\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)', @@ -325,15 +325,15 @@ def parse_sql_file_cached(filepath_str: str, file_hash: str) -> tuple: extension_name = extension_match.group(1) meta["type"] = "create extension" meta["name"] = extension_name - + chunk_text = f"File: {filepath}\nType: {meta['type']}\nName: {meta['name']}\n" if leading_comments: chunk_text += f"\nComments:\n{leading_comments}\n\n" chunk_text += f"SQL:\n{stmt_clean}" - + chunks.append({"text": chunk_text, "metadata": meta}) continue - + # Special handling for CREATE TRIGGER trigger_match = re.match( r'CREATE\s+(?:OR\s+REPLACE\s+)?TRIGGER\s+(\w+)', @@ -344,15 +344,15 @@ def parse_sql_file_cached(filepath_str: str, file_hash: str) -> tuple: trigger_name = trigger_match.group(1) meta["type"] = "create trigger" meta["name"] = trigger_name - + chunk_text = f"File: {filepath}\nType: {meta['type']}\nName: {meta['name']}\n" if leading_comments: chunk_text += f"\nComments:\n{leading_comments}\n\n" chunk_text += f"SQL:\n{stmt_clean}" - + chunks.append({"text": chunk_text, "metadata": meta}) continue - + # Try sqlglot for standard SQL parsed = sqlglot.parse_one(stmt_clean, read="postgres") stmt_type = getattr(parsed, "key", None) or parsed.token_type if hasattr(parsed, "token_type") else None @@ -360,7 +360,7 @@ def parse_sql_file_cached(filepath_str: str, file_hash: str) -> tuple: tables = [t.this for t in parsed.find_all(sqlglot.exp.Table)] meta["tables"] = ",".join([str(t) for t in tables if isinstance(t, str)]) - + ctes = [] for cte in parsed.find_all(sqlglot.exp.CTE): try: @@ -392,10 +392,10 @@ def parse_sql_file_cached(filepath_str: str, file_hash: str) -> tuple: if leading_comments: chunk_text += f"\nComments:\n{leading_comments}\n\n" chunk_text += f"SQL:\n{stmt_clean}" - + chunks.append({"text": chunk_text, "metadata": meta}) continue - + except Exception: pass @@ -412,7 +412,7 @@ def parse_sql_file_cached(filepath_str: str, file_hash: str) -> tuple: if leading_comments: chunk_text += f"\nComments:\n{leading_comments}\n\n" chunk_text += f"SQL:\n{stmt_clean}" - + chunks.append({"text": chunk_text, "metadata": meta}) return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) @@ -428,7 +428,7 @@ def parse_sql_file_cached(filepath_str: str, file_hash: str) -> tuple: statements = [s for s in sqlparse.split(sql_code) if s and s.strip()] lines = sql_code.splitlines() search_pos = 0 - + for stmt in statements: # find position of this statement in the original SQL (using running search_pos) start_idx = sql_code.find(stmt, search_pos) @@ -525,7 +525,7 @@ def parse_sql_file_cached(filepath_str: str, file_hash: str) -> tuple: "metadata": meta }) pass - + return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) def parse_sql_file(filepath: Path) -> List[Dict[str, Any]]: @@ -611,19 +611,19 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: filepath = Path(filepath_str) chunks = [] helper = Path("./tools/parse_go_ast") - + if helper.exists(): try: proc = subprocess.run( - [str(helper), str(filepath)], - capture_output=True, - text=True, - check=True, + [str(helper), str(filepath)], + capture_output=True, + text=True, + check=True, timeout=20 ) decls = json.loads(proc.stdout) lines = filepath.read_text(encoding="utf-8").splitlines(keepends=True) - + for d in decls: # Use lowercase keys (matching the Go JSON output) start = max(0, d.get("start_line", 1) - 1) @@ -632,23 +632,23 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: typ = d.get("type", "") doc_comment = d.get("doc_comment", "") receiver = d.get("receiver", "") - + # Skip package declarations if typ == "package": continue - + # Build display type display_type = typ if receiver: display_type = f"method ({receiver})" - + chunk_code = "".join(lines[start:end]) - + chunk_text = f"File: {filepath}\nType: {display_type}\nName: {name}\n" if doc_comment: chunk_text += f"Doc:\n{doc_comment}\n\n" chunk_text += f"Code:\n{chunk_code}" - + chunks.append({ "text": chunk_text, "metadata": { @@ -660,28 +660,28 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: "language": "go", } }) - + if chunks: # If helper worked, return its results return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) - + except subprocess.CalledProcessError as e: logger.warning(f"parse_go_ast failed for {filepath}: {e.stderr}; falling back to regex") except json.JSONDecodeError as e: logger.warning(f"parse_go_ast output invalid JSON for {filepath}: {e}; falling back to regex") except Exception as e: logger.warning(f"parse_go_file helper failed for {filepath}: {e}; falling back to regex") - + # Fallback: Use regex-based parsing (same as before) try: content = filepath.read_text(encoding="utf-8") lines = content.splitlines(keepends=True) - + # Pattern to match Go function declarations func_pattern = re.compile( r'^func\s+(?:\([^)]+\)\s+)?(\w+)\s*\([^)]*\)(?:\s*\([^)]*\)|\s+[\w\[\].*]+)?\s*\{', re.MULTILINE ) - + for match in func_pattern.finditer(content): func_name = match.group(1) start_pos = match.start() @@ -689,12 +689,12 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: end_line = find_brace_block_end_go(lines, start_line) func_code = "".join(lines[start_line:end_line + 1]) comments = extract_go_comments(lines, start_line) - + chunk_text = f"File: {filepath}\nType: function\nName: {func_name}\n" if comments: chunk_text += f"Comments:\n{comments}\n\n" chunk_text += f"Code:\n{func_code}" - + chunks.append({ "text": chunk_text, "metadata": { @@ -705,7 +705,7 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: "language": "go", } }) - + # Structs struct_pattern = re.compile(r'^type\s+(\w+)\s+struct\s*\{', re.MULTILINE) for match in struct_pattern.finditer(content): @@ -715,12 +715,12 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: end_line = find_brace_block_end_go(lines, start_line) struct_code = "".join(lines[start_line:end_line + 1]) comments = extract_go_comments(lines, start_line) - + chunk_text = f"File: {filepath}\nType: struct\nName: {struct_name}\n" if comments: chunk_text += f"Comments:\n{comments}\n\n" chunk_text += f"Code:\n{struct_code}" - + chunks.append({ "text": chunk_text, "metadata": { @@ -731,7 +731,7 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: "language": "go", } }) - + # Interfaces interface_pattern = re.compile(r'^type\s+(\w+)\s+interface\s*\{', re.MULTILINE) for match in interface_pattern.finditer(content): @@ -741,12 +741,12 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: end_line = find_brace_block_end_go(lines, start_line) interface_code = "".join(lines[start_line:end_line + 1]) comments = extract_go_comments(lines, start_line) - + chunk_text = f"File: {filepath}\nType: interface\nName: {interface_name}\n" if comments: chunk_text += f"Comments:\n{comments}\n\n" chunk_text += f"Code:\n{interface_code}" - + chunks.append({ "text": chunk_text, "metadata": { @@ -757,10 +757,10 @@ def parse_go_file_cached(filepath_str: str, file_hash: str) -> tuple: "language": "go", } }) - + except Exception as e: logger.warning(f"Regex-based Go parsing failed for {filepath}: {e}") - + return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) @@ -768,10 +768,10 @@ def find_brace_block_end_go(lines: List[str], start_line: int) -> int: """Find the end of a Go brace block starting at start_line.""" brace_count = 0 in_block = False - + for i in range(start_line, len(lines)): line = lines[i] - + for char in line: if char == '{': brace_count += 1 @@ -780,7 +780,7 @@ def find_brace_block_end_go(lines: List[str], start_line: int) -> int: brace_count -= 1 if in_block and brace_count == 0: return i - + return len(lines) - 1 @@ -788,10 +788,10 @@ def extract_go_comments(lines: List[str], func_line_index: int, max_lines: int = """Extract leading comments above a Go function/struct/interface.""" comments = [] idx = func_line_index - 1 - + while idx >= 0 and len(comments) < max_lines: line = lines[idx].strip() - + if line.startswith('//'): # Remove // and trim comments.insert(0, line[2:].strip()) @@ -826,13 +826,13 @@ def extract_go_comments(lines: List[str], func_line_index: int, max_lines: int = break else: break - + # Clean up while comments and comments[0] == '': comments.pop(0) while comments and comments[-1] == '': comments.pop(-1) - + return '\n'.join(comments) def parse_go_file(filepath: Path) -> List[Dict]: @@ -1136,17 +1136,17 @@ def parse_shell_file_cached(filepath_str: str, file_hash: str) -> tuple: content = filepath.read_text(encoding="utf-8") chunks = [] lines = content.splitlines(keepends=True) - + patterns = [ r'^(\w+)\s*\(\s*\)\s*\{', r'^function\s+(\w+)\s*\{', r'^function\s+(\w+)\s*\(\s*\)\s*\{', r'^def\s+(\w+)\s*\(\s*\)\s*\{', ] - + for i, line in enumerate(lines): line_text = line.strip() - + for pattern in patterns: match = re.match(pattern, line_text) if match: @@ -1154,15 +1154,15 @@ def parse_shell_file_cached(filepath_str: str, file_hash: str) -> tuple: start_line = i end_line = find_shell_function_end(lines, i) func_code = "".join(lines[start_line:end_line + 1]) - + # Extract leading comments leading_comments = extract_shell_comments(lines, start_line) - + chunk_text = f"File: {filepath}\nType: function\nName: {func_name}\n" if leading_comments: chunk_text += f"Comments:\n{leading_comments}\n\n" chunk_text += f"Code:\n{func_code}" - + chunks.append({ "text": chunk_text, "metadata": { @@ -1175,14 +1175,14 @@ def parse_shell_file_cached(filepath_str: str, file_hash: str) -> tuple: } }) break # Move to next line after finding a match - + # If no functions found, fall back to basic chunking return tuple((c["text"], tuple(c["metadata"].items())) for c in chunks) if chunks else tuple() - + except Exception as e: logger.warning(f"parse_shell_file failed {filepath}: {e}") return tuple() - + def parse_shell_file(filepath: Path) -> List[Dict]: """Enhanced shell script parsing with caching.""" file_hash = get_file_hash(filepath) @@ -1195,7 +1195,7 @@ def extract_shell_comments(lines: List[str], function_line_index: int, max_lines """Extract leading comments above a shell function.""" comments = [] idx = function_line_index - 1 - + while idx >= 0 and len(comments) < max_lines: line = lines[idx].strip() if line.startswith('#'): @@ -1210,32 +1210,32 @@ def extract_shell_comments(lines: List[str], function_line_index: int, max_lines break else: break - + # Clean up: remove leading/trailing empty lines while comments and comments[0] == "": comments.pop(0) while comments and comments[-1] == "": comments.pop(-1) - + return "\n".join(comments) - + def find_shell_function_end(lines: List[str], start_line_index: int) -> int: """ Find the end of a shell function by tracking brace nesting. - + Args: lines: List of file lines start_line_index: Starting line index (0-based) of the function - + Returns: Line index (0-based) where the function ends """ brace_count = 0 in_function = False - + for i in range(start_line_index, len(lines)): line = lines[i] - + # Count opening and closing braces for char in line: if char == '{': @@ -1243,21 +1243,21 @@ def find_shell_function_end(lines: List[str], start_line_index: int) -> int: in_function = True elif char == '}': brace_count -= 1 - + # If we've returned to brace_count 0 and we were in a function, this is the end if in_function and brace_count == 0: return i - + # Special case: shell functions without braces (single line) if not in_function and i > start_line_index: # Look for function end patterns stripped = line.strip() - if (stripped.startswith(('function ', 'def ')) or + if (stripped.startswith(('function ', 'def ')) or re.match(r'^\w+\(\s*\)', stripped) or stripped.endswith(')') and '()' not in line): # New function starting, so previous one ended return i - 1 - + # If we never found the end, return the last line return len(lines) - 1 @@ -1335,28 +1335,28 @@ def build_indexes(): global vectorstore, bm25, bm25_corpus, chunks_metadata, embeddings, index_build_time logger.info("Building indexes with contextual weighting and batch embedding...") start_time = time.time() - + texts, metadatas = index_codebase(CODEBASE_PATH) os.environ["OLLAMA_HOST"] = OLLAMA_BASE_URL embeddings = OllamaEmbeddings(model=EMBEDDING_MODEL, base_url=OLLAMA_BASE_URL) - + # Generate base embeddings in batches logger.info(f"Generating embeddings for {len(texts)} documents in batches of {EMBEDDING_BATCH_SIZE}...") base_embeddings = batch_embed_documents(texts) - + # Apply contextual weights logger.info("Applying contextual weights...") weighted_embeddings = apply_contextual_weights_to_embeddings(base_embeddings, metadatas) - + # Create vector store with weighted embeddings logger.info("Creating vector store (Chroma) with weighted embeddings...") - + # Create a wrapper that will return our pre-computed weighted embeddings class WeightedEmbeddingFunction: def __init__(self, weighted_embs): self.weighted_embs = weighted_embs self.idx = 0 - + def embed_documents(self, texts): # Return the pre-computed weighted embeddings start = self.idx @@ -1364,7 +1364,7 @@ def build_indexes(): result = self.weighted_embs[start:end] self.idx = end return result - + def batch_embed_documents(texts: List[str]) -> List[List[float]]: # Embed documents in batches all_embeddings = [] @@ -1373,29 +1373,29 @@ def build_indexes(): embeddings = embeddings.embed_documents(batch) all_embeddings.extend(embeddings) return all_embeddings - + def embed_query(self, text): # For queries, use the base embedding model (no weighting) return embeddings.embed_query(text) - + weighted_emb_func = WeightedEmbeddingFunction(weighted_embeddings) vectorstore = Chroma.from_texts( - texts=texts, - embedding=weighted_emb_func, - persist_directory=str(VECTOR_DB_PATH), + texts=texts, + embedding=weighted_emb_func, + persist_directory=str(VECTOR_DB_PATH), metadatas=metadatas ) - + # Build BM25 index logger.info("Building BM25 index...") tokenized = [t.lower().split() for t in texts] bm25 = BM25Okapi(tokenized) bm25_corpus = texts chunks_metadata = metadatas - + # Save BM25 index and metadata BM25_INDEX_PATH.write_text( - json.dumps({"corpus": texts, "metadata": metadatas}, indent=2), + json.dumps({"corpus": texts, "metadata": metadatas}, indent=2), encoding="utf-8" ) index_build_time = time.time() @@ -1409,14 +1409,14 @@ def batch_embed_documents(texts: List[str]) -> List[List[float]]: """Embed documents in batches for better performance.""" all_embeddings = [] total_batches = (len(texts) + EMBEDDING_BATCH_SIZE - 1) // EMBEDDING_BATCH_SIZE - + for i in range(0, len(texts), EMBEDDING_BATCH_SIZE): batch = texts[i:i + EMBEDDING_BATCH_SIZE] batch_num = i // EMBEDDING_BATCH_SIZE + 1 logger.info(f"Embedding batch {batch_num}/{total_batches} ({len(batch)} docs)...") batch_embeddings = embeddings.embed_documents(batch) all_embeddings.extend(batch_embeddings) - + return all_embeddings def load_indexes(): @@ -1509,21 +1509,21 @@ def rerank_with_ollama_enhanced(query: str, candidates: List[str], top_k: int = """Enhanced Qwen3 Reranker with batch processing and better scoring.""" if not candidates: return [] - + # Use domain-specific instruction for code search instruction = "Given a technical code search query, retrieve relevant code implementations, function definitions, or examples that directly address the query requirements" - + batch_scores = [] - + # Process in smaller batches to avoid timeouts batch_size = 5 for i in range(0, len(candidates), batch_size): batch = candidates[i:i + batch_size] batch_prompts = [] - + for chunk in batch: system_prompt = '<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>\n' - + user_prompt = ( f'<|im_start|>user\n' f': {instruction}\n' @@ -1532,9 +1532,9 @@ def rerank_with_ollama_enhanced(query: str, candidates: List[str], top_k: int = f'<|im_end|>\n' f'<|im_start|>assistant\n\n\n\n\n' ) - + batch_prompts.append(system_prompt + user_prompt) - + # Score batch for prompt, chunk in zip(batch_prompts, batch): try: @@ -1549,21 +1549,21 @@ def rerank_with_ollama_enhanced(query: str, candidates: List[str], top_k: int = }, timeout=20 ) - + j = resp.json() response_text = j.get("response", "").strip().lower() - + # Enhanced scoring with confidence levels score = parse_reranker_response(response_text) batch_scores.append((chunk, score)) - + except requests.exceptions.Timeout: logger.warning("Reranker request timeout, assigning default score") batch_scores.append((chunk, 0.0)) except Exception as e: logger.warning(f"Rerank call failed: {e}") batch_scores.append((chunk, 0.0)) - + # Sort by score descending and return top_k batch_scores.sort(key=lambda x: x[1], reverse=True) return batch_scores[:top_k] @@ -1572,20 +1572,20 @@ def rerank_with_ollama_enhanced(query: str, candidates: List[str], top_k: int = def parse_reranker_response(response_text: str) -> float: """Parse Qwen3 Reranker response and convert to confidence score.""" response_lower = response_text.strip().lower() - + # Exact matches from the model if response_lower == "yes": return 1.0 elif response_lower == "no": return 0.0 - + # Handle variations and partial matches yes_indicators = ["yes", "relevant", "correct", "matches", "appropriate", "suitable"] no_indicators = ["no", "irrelevant", "incorrect", "unrelated", "inappropriate"] - + yes_count = sum(1 for indicator in yes_indicators if indicator in response_lower) no_count = sum(1 for indicator in no_indicators if indicator in response_lower) - + if yes_count > no_count: return 0.8 # Likely relevant but not confident elif no_count > yes_count: @@ -1685,7 +1685,7 @@ def read_file_lines(path: str, start: int = 1, end: Optional[int] = None) -> str p = Path(path) else: p = CODEBASE_PATH / Path(path) - + # Security: Validate path is within codebase try: resolved = p.resolve() @@ -1694,51 +1694,51 @@ def read_file_lines(path: str, start: int = 1, end: Optional[int] = None) -> str return f"Error: Path '{path}' is outside the codebase directory" except (ValueError, OSError) as e: return f"Error: Invalid path '{path}': {e}" - + try: content = p.read_text(encoding="utf-8") lines = content.splitlines() - + if end is None: end = start + 50 # reasonable default - + # Clamp to valid range start_i = max(1, start) end_i = min(len(lines), end) - + # Detect language language = detect_language(p) - + # Build context-aware line range context_start, context_end, context_info = find_context_boundaries( lines, start_i, end_i, language ) - + # Extract the context lines context_lines = lines[context_start - 1:context_end] - + # Build output with metadata output = [] output.append(f"File: {path}") output.append(f"Language: {language}") output.append(f"Requested: lines {start_i}-{end_i}") output.append(f"Showing: lines {context_start}-{context_end} (with context)") - + if context_info: output.append(f"Context: {context_info}") - + output.append("\n" + "="*60) - + # Add line numbers for i, line in enumerate(context_lines, start=context_start): # Highlight the originally requested range marker = ">>> " if start_i <= i <= end_i else " " output.append(f"{marker}{i:4d} | {line}") - + output.append("="*60) - + return "\n".join(output) - + except Exception as e: return f"Failed to read {path}: {e}" @@ -1751,50 +1751,50 @@ def find_context_boundaries(lines: List[str], start: int, end: int, language: st # Convert to 0-based indexing for processing start_idx = start - 1 end_idx = end - 1 - + context_info = [] - + # Expand upward to include leading comments and function/class headers context_start = start - + # Step 1: Include leading comments above start comment_start = find_leading_comments_start(lines, start_idx, language) if comment_start < start_idx: context_start = comment_start + 1 context_info.append("leading comments") - + # Step 2: Check if we're inside a function/class and include its signature func_start, func_end, func_name, func_type = find_enclosing_function_or_class( lines, start_idx, end_idx, language ) - + if func_start is not None: context_start = min(context_start, func_start + 1) if func_end is not None and func_end > end_idx: context_end = func_end + 1 else: context_end = end - + if func_name: context_info.append(f"{func_type} '{func_name}'") else: context_end = end - + # Step 3: Include decorators (Python) or annotations (Java) if language == "python": decorator_start = find_decorators_start(lines, context_start - 1) if decorator_start < context_start - 1: context_start = decorator_start + 1 context_info.append("decorators") - + # Limit expansion to reasonable bounds (max 100 lines of context) max_context = 100 if context_end - context_start > max_context: context_end = context_start + max_context context_info.append("truncated to 100 lines") - + info_str = " + ".join(context_info) if context_info else "no additional context" - + return context_start, context_end, info_str @@ -1809,18 +1809,18 @@ def find_leading_comments_start(lines: List[str], line_idx: int, language: str) "sql": ("--", "/*"), "rust": ("//", "/*"), } - + patterns = comment_patterns.get(language, ("#", "//", "/*", "--")) - + start_idx = line_idx i = line_idx - 1 - + # Track if we're in a block comment in_block = False - + while i >= 0: line = lines[i].strip() - + # Empty lines are OK if we already have comments if not line: if i < line_idx - 1: # Allow one blank line between comments @@ -1828,30 +1828,30 @@ def find_leading_comments_start(lines: List[str], line_idx: int, language: str) continue else: break - + # Check for block comment end (we're going backwards) if "*/" in line: in_block = True start_idx = i i -= 1 continue - + # Check for block comment start if in_block and ("/*" in line or "/**" in line): start_idx = i in_block = False i -= 1 continue - + # Check for single-line comments is_comment = any(line.startswith(p) for p in patterns) - + if is_comment or in_block: start_idx = i i -= 1 else: break - + return start_idx @@ -1884,7 +1884,7 @@ def find_enclosing_function_or_class(lines: List[str], start_idx: int, end_idx: return find_java_method_or_class(lines, start_idx, end_idx) elif language == "sql": return find_sql_function_or_block(lines, start_idx, end_idx) - + return None, None, None, None @@ -1892,14 +1892,14 @@ def find_python_function_or_class(lines: List[str], start_idx: int, end_idx: int """Find enclosing Python function or class.""" # Look backwards for def/class with proper indentation target_indent = None - + for i in range(start_idx, -1, -1): line = lines[i] stripped = line.lstrip() - + if stripped.startswith(("def ", "class ", "async def ")): indent = len(line) - len(stripped) - + # Check if this could be our enclosing scope if target_indent is None or indent < target_indent: # Extract name @@ -1907,12 +1907,12 @@ def find_python_function_or_class(lines: List[str], start_idx: int, end_idx: int if match: name = match.group(1) func_type = "class" if stripped.startswith("class") else "function" - + # Find the end by looking for next same-or-lower indent non-empty line end_i = find_python_block_end(lines, i, indent) - + return i, end_i, name, func_type - + return None, None, None, None @@ -1921,15 +1921,15 @@ def find_python_block_end(lines: List[str], start_idx: int, base_indent: int) -> for i in range(start_idx + 1, len(lines)): line = lines[i] stripped = line.lstrip() - + if not stripped: # Skip empty lines continue - + indent = len(line) - len(stripped) - + if indent <= base_indent and stripped and not stripped.startswith("#"): return i - 1 - + return len(lines) - 1 @@ -1937,7 +1937,7 @@ def find_js_function(lines: List[str], start_idx: int, end_idx: int) -> Tuple[Op """Find enclosing JavaScript/TypeScript function.""" for i in range(start_idx, -1, -1): line = lines[i].strip() - + # Match various JS function patterns patterns = [ r'function\s+(\w+)', @@ -1945,14 +1945,14 @@ def find_js_function(lines: List[str], start_idx: int, end_idx: int) -> Tuple[Op r'(\w+)\s*\([^)]*\)\s*{', # Arrow functions r'async\s+function\s+(\w+)', ] - + for pattern in patterns: match = re.search(pattern, line) if match: name = match.group(1) end_i = find_brace_block_end(lines, i) return i, end_i, name, "function" - + return None, None, None, None @@ -1960,14 +1960,14 @@ def find_go_function(lines: List[str], start_idx: int, end_idx: int) -> Tuple[Op """Find enclosing Go function.""" for i in range(start_idx, -1, -1): line = lines[i].strip() - + # Match Go function: func (receiver) name(params) returnType { match = re.match(r'func\s+(?:\([^)]+\)\s+)?(\w+)', line) if match: name = match.group(1) end_i = find_brace_block_end(lines, i) return i, end_i, name, "function" - + return None, None, None, None @@ -1975,21 +1975,21 @@ def find_java_method_or_class(lines: List[str], start_idx: int, end_idx: int) -> """Find enclosing Java method or class.""" for i in range(start_idx, -1, -1): line = lines[i].strip() - + # Match class class_match = re.match(r'(?:public|private|protected)?\s*(?:static)?\s*class\s+(\w+)', line) if class_match: name = class_match.group(1) end_i = find_brace_block_end(lines, i) return i, end_i, name, "class" - + # Match method method_match = re.match(r'(?:public|private|protected)?\s*(?:static)?\s*(?:\w+(?:<[^>]+>)?)\s+(\w+)\s*\(', line) if method_match: name = method_match.group(1) end_i = find_brace_block_end(lines, i) return i, end_i, name, "method" - + return None, None, None, None @@ -1997,7 +1997,7 @@ def find_sql_function_or_block(lines: List[str], start_idx: int, end_idx: int) - """Find enclosing SQL function or procedure.""" for i in range(start_idx, -1, -1): line = lines[i].strip().upper() - + if line.startswith(("CREATE FUNCTION", "CREATE OR REPLACE FUNCTION", "CREATE PROCEDURE")): # Extract name match = re.search(r'(?:FUNCTION|PROCEDURE)\s+(\w+)', line, re.IGNORECASE) @@ -2007,7 +2007,7 @@ def find_sql_function_or_block(lines: List[str], start_idx: int, end_idx: int) - end_i = find_sql_function_end(lines, i) func_type = "procedure" if "PROCEDURE" in line else "function" return i, end_i, name, func_type - + return None, None, None, None @@ -2024,20 +2024,20 @@ def find_brace_block_end(lines: List[str], start_idx: int) -> Optional[int]: """Find the end of a brace-delimited block {}.""" brace_count = 0 started = False - + for i in range(start_idx, len(lines)): line = lines[i] - + for char in line: if char == '{': brace_count += 1 started = True elif char == '}': brace_count -= 1 - + if started and brace_count == 0: return i - + return len(lines) - 1 # ----------------------------- @@ -2045,9 +2045,9 @@ def find_brace_block_end(lines: List[str], start_idx: int) -> Optional[int]: # ----------------------------- @mcp.tool() def health_check() -> str: - """What it does – Quick JSON status of the RAG server (ready, indexed path, chunk count, age). -When to use – Before any other query, to confirm the index is current. -When not to use – After you already know the server is healthy; it adds no value. + """What it does – Quick JSON status of the RAG server (ready, indexed path, chunk count, age). +When to use – Before any other query, to confirm the index is current. +When not to use – After you already know the server is healthy; it adds no value. Example – health_check() → { "status":"ready","total_chunks":11234,"index_age_hours":4.7 }""" try: with _startup_lock: @@ -2063,7 +2063,7 @@ Example – health_check() → { "status":"ready","total_chunks":11234,"index_ag "batch_size": EMBEDDING_BATCH_SIZE } } - + if bm25_corpus: status["statistics"] = { "total_chunks": len(bm25_corpus), @@ -2072,7 +2072,7 @@ Example – health_check() → { "status":"ready","total_chunks":11234,"index_ag "avg_chunk_size_chars": round(calculate_avg_chunk_size(bm25_corpus), 1), "index_age_hours": round((time.time() - index_build_time) / 3600, 1) if index_build_time > 0 else None } - + return json.dumps(status, indent=2) except Exception as e: return json.dumps({"status": "error", "message": str(e)}, indent=2) @@ -2086,12 +2086,12 @@ Rerank false is good enough for 65% of searches and is VERY fast, rerank true is Example – search_codebase("user authentication python", top_k=5, rerank=True) → 5 relevant snippets.""" try: hy = hybrid_search(query, k=max(top_k, RERANK_TOP_N)) - + if ENABLE_RERANK and rerank: try: candidates = [t for t, m, s in hy] rr = rerank_with_ollama_enhanced(query, candidates[:RERANK_TOP_N], top_k=top_k) - + # Build clean TOON format directly results = [] for i, (chunk_text, score) in enumerate(rr, 1): @@ -2103,9 +2103,9 @@ Example – search_codebase("user authentication python", top_k=5, rerank=True) 'score': score, 'content': _extract_clean_content(chunk_text) }) - + return _format_toon_results(results, query, "search") - + except Exception as e: logger.warning(f"Rerank step failed: {e}") # Fall through to non-reranked results @@ -2120,9 +2120,9 @@ Example – search_codebase("user authentication python", top_k=5, rerank=True) 'score': score, 'content': _extract_clean_content(text) }) - + return _format_toon_results(results, query, "search") - + except RuntimeError as e: return f"Error: {e}" except Exception as e: @@ -2134,7 +2134,7 @@ def _extract_clean_content(chunk_text: str) -> str: """Extract clean content from chunk text by removing duplicate metadata.""" lines = chunk_text.split('\n') content_lines = [] - + # Skip the first few metadata lines (File:, Type:, Name:, etc.) skip_metadata = True for line in lines: @@ -2148,7 +2148,7 @@ def _extract_clean_content(chunk_text: str) -> str: content_lines.append(stripped) else: content_lines.append(stripped) - + # Join and clean up content = ' '.join(content_lines) content = re.sub(r'\s+', ' ', content) # Normalize whitespace @@ -2159,9 +2159,9 @@ def _format_toon_results(results: List[Dict], query: str, result_type: str) -> s """Format results in clean TOON format.""" if not results: return f"# {result_type.title()}: {query}\nNo results found.\n" - + lines = [f"# {result_type.title()}: {query}", f"results[{len(results)}]{{file,line,score,content}}:"] - + for result in results: row = [ result.get('file', ''), @@ -2169,7 +2169,7 @@ def _format_toon_results(results: List[Dict], query: str, result_type: str) -> s f"{result.get('score', 0.0):.2f}", result.get('content', '')[:500] # Reasonable limit ] - + # Escape fields if needed escaped_row = [] for field in row: @@ -2179,9 +2179,9 @@ def _format_toon_results(results: List[Dict], query: str, result_type: str) -> s escaped_row.append(f'"{escaped}"') else: escaped_row.append(field_str) - + lines.append(" " + ",".join(escaped_row)) - + return "\n".join(lines) @@ -2204,7 +2204,7 @@ Example – find_code_references("AuthService") → list of references.""" 'line': m['line'], 'context': m['context'] }) - + return _format_reference_results(results, symbol) @@ -2212,16 +2212,16 @@ def _format_reference_results(results: List[Dict], symbol: str) -> str: """Format reference results in clean TOON format.""" if not results: return f"# References: {symbol}\nNo references found.\n" - + lines = [f"# References: {symbol}", f"references[{len(results)}]{{file,line,context}}:"] - + for result in results: row = [ result.get('file', ''), result.get('line', ''), result.get('context', '')[:300] # Reasonable limit ] - + # Escape fields if needed escaped_row = [] for field in row: @@ -2231,9 +2231,9 @@ def _format_reference_results(results: List[Dict], symbol: str) -> str: escaped_row.append(f'"{escaped}"') else: escaped_row.append(field_str) - + lines.append(" " + ",".join(escaped_row)) - + return "\n".join(lines) @mcp.tool() @@ -2260,7 +2260,7 @@ Example – rebuild_index() → '✅ Index rebuilt (13.2 s); 11,234 chunks'""" parse_java_file_cached.cache_clear() parse_svelte_file_cached.cache_clear() parse_shell_file_cached.cache_clear() - + t0 = time.time() build_indexes() dt = time.time() - t0 @@ -2291,7 +2291,7 @@ def startup(): def signal_handler(sig, frame): """Handle graceful shutdown on Ctrl+C.""" logger.info("\n=== Shutdown signal received ===") - + # Clear any LRU caches to release memory try: parse_python_file_cached.cache_clear() @@ -2303,11 +2303,11 @@ def signal_handler(sig, frame): logger.info("✓ Caches cleared") except Exception as e: logger.warning(f"Cache clearing error: {e}") - + # Note: Chroma auto-persists, no explicit close needed logger.info("✓ Indexes are already persisted to disk") logger.info("Goodbye!\n") - + # Force exit immediately to avoid thread hang os._exit(0) @@ -2315,17 +2315,17 @@ if __name__ == "__main__": # Register signal handlers signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) - + try: startup() logger.info("="*60) logger.info("🚀 MCP RAG Server is ready!") logger.info("Use 'python serve_http.py' for HTTP server") logger.info("Press Ctrl+C to stop") - + # Just run in stdio mode by default mcp.run(transport='stdio') - + except KeyboardInterrupt: signal_handler(signal.SIGINT, None) except Exception as e: