filter valid chunks

huggingface · Jun 25, 2024 · d5fb907 · d5fb907
1 parent 7582341
commit d5fb907
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/doc_builder/build_embeddings.py b/src/doc_builder/build_embeddings.py
@@ -463,6 +463,7 @@ def build_embeddings(
     # return
 
     # Step 2: create embeddings
+    chunks = [chunk for chunk in chunks if chunk.text and chunk.source and chunk.package_name]  # filter valid chunks
     embeddings = call_embedding_inference(chunks, hf_ie_name, hf_ie_namespace, hf_ie_token)
 
     # Step 3: push embeddings to vector database (meilisearch)