From d7fb2d6938056a76846d584416d4d1c44b8657a9 Mon Sep 17 00:00:00 2001 From: Stephen Adamson Date: Tue, 4 Nov 2025 23:13:18 +0000 Subject: [PATCH] Revert "remove unused batch function" This reverts commit 6f68d471fa1f4cf625a1013e6bbf4c807f42ec9d. --- mcp_codebase.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mcp_codebase.py b/mcp_codebase.py index 011c792..61d64e1 100644 --- a/mcp_codebase.py +++ b/mcp_codebase.py @@ -1401,6 +1401,23 @@ def build_indexes(): elapsed = index_build_time - start_time logger.info(f"Index build complete with contextual weighting applied. Total time: {elapsed:.1f}s") +# ----------------------------- +# Build / load indexes with contextual weighting and batch embedding +# ----------------------------- +def batch_embed_documents(texts: List[str]) -> List[List[float]]: + """Embed documents in batches for better performance.""" + all_embeddings = [] + total_batches = (len(texts) + EMBEDDING_BATCH_SIZE - 1) // EMBEDDING_BATCH_SIZE + + for i in range(0, len(texts), EMBEDDING_BATCH_SIZE): + batch = texts[i:i + EMBEDDING_BATCH_SIZE] + batch_num = i // EMBEDDING_BATCH_SIZE + 1 + logger.info(f"Embedding batch {batch_num}/{total_batches} ({len(batch)} docs)...") + batch_embeddings = embeddings.embed_documents(batch) + all_embeddings.extend(batch_embeddings) + + return all_embeddings + def load_indexes(): global vectorstore, bm25, bm25_corpus, chunks_metadata, embeddings, index_build_time logger.info("Loading indexes from disk...")