redo WeightedEmbeddingFunction

mcp_codebase.py

@@ -2133,32 +2133,44 @@ def build_indexes():
 
     # Create a wrapper that will return our pre-computed weighted embeddings
     class WeightedEmbeddingFunction:
-        def __init__(self, weighted_embs):
-            self.weighted_embs = weighted_embs
-            self.idx = 0
+        def __init__(self, texts, weighted_embs):
+            """
+            texts:          list[str]
+            weighted_embs:  list[list[float]]
+            """
 
-        def embed_documents(self, texts):
-            # Return the pre-computed weighted embeddings
-            start = self.idx
-            end = start + len(texts)
-            result = self.weighted_embs[start:end]
-            self.idx = end
+            # Map each text to a queue of its embeddings (duplicate-safe)
+            self.lookup = defaultdict(deque)
+
+            for t, emb in zip(texts, weighted_embs):
+                self.lookup[t].append(emb)
+
+        def embed_documents(self, docs):
+            """
+            Chroma may pass docs in any size or order.
+            Always return correct per-document embeddings.
+            """
+            result = []
+            for d in docs:
+                if not self.lookup[d]:
+                    raise ValueError(
+                        f"No embedding left for text {repr(d[:80])} — "
+                        f"likely mismatch between texts and embeddings."
+                    )
+                result.append(self.lookup[d].popleft())
             return result
 
-        def batch_embed_documents(texts: List[str]) -> List[List[float]]:
-            # Embed documents in batches
-            all_embeddings = []
-            for i in range(0, len(texts), EMBEDDING_BATCH_SIZE):
-                batch = texts[i:i + EMBEDDING_BATCH_SIZE]
-                embeddings = embeddings.embed_documents(batch)
-                all_embeddings.extend(embeddings)
-            return all_embeddings
+        def embed_query(self, q):
+            """
+            Queries should use base model, not weighted context embeddings.
+            """
+            return embeddings.embed_query(q)
 
-        def embed_query(self, text):
-            # For queries, use the base embedding model (no weighting)
-            return embeddings.embed_query(text)
+    weighted_emb_func = WeightedEmbeddingFunction(
+        texts=texts,
+        weighted_embs=weighted_embeddings
+    )
 
-    weighted_emb_func = WeightedEmbeddingFunction(weighted_embeddings)
     vectorstore = Chroma.from_texts(
         texts=texts,
         embedding=weighted_emb_func,