better chunks

huggingface · Jun 26, 2024 · fef86b7 · fef86b7
1 parent 548fe29
commit fef86b7
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/src/doc_builder/build_embeddings.py b/src/doc_builder/build_embeddings.py
@@ -30,7 +30,7 @@
 from .convert_md_to_mdx import process_md
 from .convert_rst_to_mdx import find_indent, is_empty_line
 from .meilisearch_helper import add_embeddings_to_db, create_embedding_db, delete_embedding_db, swap_indexes
-from .utils import read_doc_config
+from .utils import chunk_list, read_doc_config
 
 
 Chunk = namedtuple("Chunk", "text source package_name")
@@ -462,7 +462,8 @@ def build_embeddings(
 
     # Step 3: push embeddings to vector database (meilisearch)
     client = meilisearch.Client("https://edge.meilisearch.com", meilisearch_key)
-    add_embeddings_to_db(client, MEILI_INDEX_TEMP, embeddings)
+    for chunk_embeddings in tqdm(chunk_list(embeddings), desc="Uploading data to meilisearch"):
+        add_embeddings_to_db(client, MEILI_INDEX_TEMP, chunk_embeddings)
 
 
 def clean_meilisearch(meilisearch_key: str):