Fix opening bytes index with dynamic codec (#1279)

* Fix opening bytes index with dynamic codec Fix #1278 * extend proptest to cover bytes field codec bug
quickwit-oss · Feb 18, 2022 · cef1457 · cef1457
1 parent e05e2a0
commit cef1457
Show file tree

Hide file tree

Showing 5 changed files with 14 additions and 11 deletions.
diff --git a/src/fastfield/bytes/reader.rs b/src/fastfield/bytes/reader.rs
@@ -1,5 +1,5 @@
 use crate::directory::{FileSlice, OwnedBytes};
-use crate::fastfield::{BitpackedFastFieldReader, FastFieldReader, MultiValueLength};
+use crate::fastfield::{DynamicFastFieldReader, FastFieldReader, MultiValueLength};
 use crate::DocId;
 
 /// Reader for byte array fast fields
@@ -14,13 +14,13 @@ use crate::DocId;
 /// and the start index for the next document, and keeping the bytes in between.
 #[derive(Clone)]
 pub struct BytesFastFieldReader {
-    idx_reader: BitpackedFastFieldReader<u64>,
+    idx_reader: DynamicFastFieldReader<u64>,
     values: OwnedBytes,
 }
 
 impl BytesFastFieldReader {
     pub(crate) fn open(
-        idx_reader: BitpackedFastFieldReader<u64>,
+        idx_reader: DynamicFastFieldReader<u64>,
         values_file: FileSlice,
     ) -> crate::Result<BytesFastFieldReader> {
         let values = values_file.read_bytes()?;

diff --git a/src/fastfield/mod.rs b/src/fastfield/mod.rs
@@ -26,7 +26,6 @@ pub use self::bytes::{BytesFastFieldReader, BytesFastFieldWriter};
 pub use self::error::{FastFieldNotAvailableError, Result};
 pub use self::facet_reader::FacetReader;
 pub use self::multivalued::{MultiValuedFastFieldReader, MultiValuedFastFieldWriter};
-pub(crate) use self::reader::BitpackedFastFieldReader;
 pub use self::reader::{DynamicFastFieldReader, FastFieldReader};
 pub use self::readers::FastFieldReaders;
 pub use self::serializer::{CompositeFastFieldSerializer, FastFieldDataAccess, FastFieldStats};

diff --git a/src/fastfield/reader.rs b/src/fastfield/reader.rs
@@ -248,8 +248,6 @@ impl<Item: FastValue, C: FastFieldCodecReader + Clone> FastFieldReader<Item>
     }
 }
 
-pub(crate) type BitpackedFastFieldReader<Item> = FastFieldReaderCodecWrapper<Item, BitpackedReader>;
-
 impl<Item: FastValue> From<Vec<Item>> for DynamicFastFieldReader<Item> {
     fn from(vals: Vec<Item>) -> DynamicFastFieldReader<Item> {
         let mut schema_builder = Schema::builder();

diff --git a/src/fastfield/readers.rs b/src/fastfield/readers.rs
@@ -1,8 +1,7 @@
 use super::reader::DynamicFastFieldReader;
 use crate::directory::{CompositeFile, FileSlice};
 use crate::fastfield::{
-    BitpackedFastFieldReader, BytesFastFieldReader, FastFieldNotAvailableError, FastValue,
-    MultiValuedFastFieldReader,
+    BytesFastFieldReader, FastFieldNotAvailableError, FastValue, MultiValuedFastFieldReader,
 };
 use crate::schema::{Cardinality, Field, FieldType, Schema};
 use crate::space_usage::PerFieldSpaceUsage;
@@ -219,7 +218,7 @@ impl FastFieldReaders {
                 )));
             }
             let fast_field_idx_file = self.fast_field_data(field, 0)?;
-            let idx_reader = BitpackedFastFieldReader::open(fast_field_idx_file)?;
+            let idx_reader = DynamicFastFieldReader::open(fast_field_idx_file)?;
             let data = self.fast_field_data(field, 1)?;
             BytesFastFieldReader::open(idx_reader, data)
         } else {

diff --git a/src/indexer/index_writer.rs b/src/indexer/index_writer.rs
@@ -1389,6 +1389,7 @@ mod tests {
     ) -> crate::Result<()> {
         let mut schema_builder = schema::Schema::builder();
         let id_field = schema_builder.add_u64_field("id", FAST | INDEXED | STORED);
+        let bytes_field = schema_builder.add_bytes_field("bytes", FAST | INDEXED | STORED);
         let text_field = schema_builder.add_text_field(
             "text_field",
             TextOptions::default()
@@ -1435,8 +1436,14 @@ mod tests {
             match op {
                 IndexingOp::AddDoc { id } => {
                     let facet = Facet::from(&("/cola/".to_string() + &id.to_string()));
-                    index_writer
-                        .add_document(doc!(id_field=>id, multi_numbers=> id, multi_numbers => id, text_field => id.to_string(), facet_field => facet, large_text_field=> LOREM))?;
+                    index_writer.add_document(doc!(id_field=>id,
+                            bytes_field => id.to_le_bytes().as_slice(),
+                            multi_numbers=> id,
+                            multi_numbers => id,
+                            text_field => id.to_string(),
+                            facet_field => facet,
+                            large_text_field=> LOREM
+                    ))?;
                 }
                 IndexingOp::DeleteDoc { id } => {
                     index_writer.delete_term(Term::from_field_u64(id_field, id));