fix large index size (#23)

facebookresearch · Nov 23, 2023 · 23639ea · 23639ea
1 parent 958b6cb
commit 23639ea
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/index.py b/src/index.py
@@ -82,7 +82,7 @@ def save_index(self, path: str, total_saved_shards: int, overwrite_saved_passage
                 passage_shard = [self.doc_map[i] for i in range(shard_start, shard_end)]
                 with open(passage_shard_path, "wb") as fobj:
                     pickle.dump(passage_shard, fobj, protocol=pickle.HIGHEST_PROTOCOL)
-            embeddings_shard = self.embeddings[:, shard_start:shard_end]
+            embeddings_shard = self.embeddings[:, shard_start:shard_end].clone()
             embedding_shard_path = self._get_saved_embedding_path(path, shard_id)
             torch.save(embeddings_shard, embedding_shard_path)