Issue/922b (#1233)

* Add a NORMED options on field

Make fieldnorm indexation optional:

* for all types except text => added a NORMED options
* for text field
** if STRING, field has not fieldnorm retained
** if TEXT, field has fieldnorm computed

* Finalize making fieldnorm optional for all field types.

- Using Option for fieldnorm readers.

CHANGELOG.md

@@ -4,6 +4,7 @@ Tantivy 0.17
 - Facets are necessarily indexed. Existing index with indexed facets should work out of the box. Index without facets that are marked with index: false should be broken (but they were already broken in a sense). (@fulmicoton) #1195 .
 - Bugfix that could in theory impact durability in theory on some filesystems [#1224](https://github.com/quickwit-inc/tantivy/issues/1224)
 - Reduce the number of fsync calls [#1225](https://github.com/quickwit-inc/tantivy/issues/1225)
+- Schema now offers not indexing fieldnorms (@lpouget) [#922](https://github.com/quickwit-inc/tantivy/issues/922)
 
 Tantivy 0.16.2
 ================================

src/core/index_meta.rs

@@ -394,7 +394,7 @@ mod tests {
         let json = serde_json::ser::to_string(&index_metas).expect("serialization failed");
         assert_eq!(
             json,
-            r#"{"index_settings":{"sort_by_field":{"field":"text","order":"Asc"},"docstore_compression":"lz4"},"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","tokenizer":"default"},"stored":false}}],"opstamp":0}"#
+            r#"{"index_settings":{"sort_by_field":{"field":"text","order":"Asc"},"docstore_compression":"lz4"},"segments":[],"schema":[{"name":"text","type":"text","options":{"indexing":{"record":"position","fieldnorms":true,"tokenizer":"default"},"stored":false}}],"opstamp":0}"#
         );
     }
 }

src/core/segment_reader.rs

@@ -127,13 +127,17 @@ impl SegmentReader {
         self.fieldnorm_readers.get_field(field)?.ok_or_else(|| {
             let field_name = self.schema.get_field_name(field);
             let err_msg = format!(
-                "Field norm not found for field {:?}. Was it marked as indexed during indexing?",
+                "Field norm not found for field {:?}. Was the field set to record norm during indexing?",
                 field_name
             );
             crate::TantivyError::SchemaError(err_msg)
         })
     }
 
+    pub(crate) fn fieldnorms_readers(&self) -> &FieldNormReaders {
+        &self.fieldnorm_readers
+    }
+
     /// Accessor to the segment's `StoreReader`.
     pub fn get_store_reader(&self) -> io::Result<StoreReader> {
         StoreReader::open(self.store_file.clone())

src/fastfield/multivalued/mod.rs

@@ -68,6 +68,7 @@ mod tests {
             IntOptions::default()
                 .set_fast(Cardinality::MultiValues)
                 .set_indexed()
+                .set_fieldnorm()
                 .set_stored(),
         );
         let time_i =

src/fieldnorm/mod.rs

@@ -26,3 +26,137 @@ pub use self::serializer::FieldNormsSerializer;
 pub use self::writer::FieldNormsWriter;
 
 use self::code::{fieldnorm_to_id, id_to_fieldnorm};
+
+#[cfg(test)]
+mod tests {
+    use crate::directory::CompositeFile;
+    use crate::directory::{Directory, RamDirectory, WritePtr};
+    use crate::fieldnorm::FieldNormReader;
+    use crate::fieldnorm::FieldNormsSerializer;
+    use crate::fieldnorm::FieldNormsWriter;
+    use crate::query::Query;
+    use crate::query::TermQuery;
+    use crate::schema::IndexRecordOption;
+    use crate::schema::TextFieldIndexing;
+    use crate::schema::TextOptions;
+    use crate::schema::TEXT;
+    use crate::Index;
+    use crate::Term;
+    use crate::TERMINATED;
+    use once_cell::sync::Lazy;
+    use std::path::Path;
+
+    use crate::schema::{Field, Schema, STORED};
+
+    pub static SCHEMA: Lazy<Schema> = Lazy::new(|| {
+        let mut schema_builder = Schema::builder();
+        schema_builder.add_text_field("field", STORED);
+        schema_builder.add_text_field("txt_field", TEXT);
+        schema_builder.add_text_field(
+            "str_field",
+            TextOptions::default().set_indexing_options(
+                TextFieldIndexing::default()
+                    .set_index_option(IndexRecordOption::Basic)
+                    .set_fieldnorms(false),
+            ),
+        );
+        schema_builder.build()
+    });
+
+    pub static FIELD: Lazy<Field> = Lazy::new(|| SCHEMA.get_field("field").unwrap());
+    pub static TXT_FIELD: Lazy<Field> = Lazy::new(|| SCHEMA.get_field("txt_field").unwrap());
+    pub static STR_FIELD: Lazy<Field> = Lazy::new(|| SCHEMA.get_field("str_field").unwrap());
+
+    #[test]
+    #[should_panic(expected = "Cannot register a given fieldnorm twice")]
+    pub fn test_should_panic_when_recording_fieldnorm_twice_for_same_doc() {
+        let mut fieldnorm_writers = FieldNormsWriter::for_schema(&SCHEMA);
+        fieldnorm_writers.record(0u32, *TXT_FIELD, 5);
+        fieldnorm_writers.record(0u32, *TXT_FIELD, 3);
+    }
+
+    #[test]
+    pub fn test_fieldnorm() -> crate::Result<()> {
+        let path = Path::new("test");
+        let directory: RamDirectory = RamDirectory::create();
+        {
+            let write: WritePtr = directory.open_write(Path::new("test"))?;
+            let serializer = FieldNormsSerializer::from_write(write)?;
+            let mut fieldnorm_writers = FieldNormsWriter::for_schema(&SCHEMA);
+            fieldnorm_writers.record(2u32, *TXT_FIELD, 5);
+            fieldnorm_writers.record(3u32, *TXT_FIELD, 3);
+            fieldnorm_writers.serialize(serializer, None)?;
+        }
+        let file = directory.open_read(&path)?;
+        {
+            let fields_composite = CompositeFile::open(&file)?;
+            assert!(fields_composite.open_read(*FIELD).is_none());
+            assert!(fields_composite.open_read(*STR_FIELD).is_none());
+            let data = fields_composite.open_read(*TXT_FIELD).unwrap();
+            let fieldnorm_reader = FieldNormReader::open(data)?;
+            assert_eq!(fieldnorm_reader.fieldnorm(0u32), 0u32);
+            assert_eq!(fieldnorm_reader.fieldnorm(1u32), 0u32);
+            assert_eq!(fieldnorm_reader.fieldnorm(2u32), 5u32);
+            assert_eq!(fieldnorm_reader.fieldnorm(3u32), 3u32);
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_fieldnorm_disabled() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let text_options = TextOptions::default()
+            .set_indexing_options(TextFieldIndexing::default().set_fieldnorms(false));
+        let text = schema_builder.add_text_field("text", text_options);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut writer = index.writer_for_tests()?;
+        writer.add_document(doc!(text=>"hello"))?;
+        writer.add_document(doc!(text=>"hello hello hello"))?;
+        writer.commit()?;
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+        let query = TermQuery::new(
+            Term::from_field_text(text, "hello"),
+            IndexRecordOption::WithFreqs,
+        );
+        let weight = query.weight(&*searcher, true)?;
+        let mut scorer = weight.scorer(searcher.segment_reader(0), 1.0f32)?;
+        assert_eq!(scorer.doc(), 0);
+        assert!((scorer.score() - 0.22920431).abs() < 0.001f32);
+        assert_eq!(scorer.advance(), 1);
+        assert_eq!(scorer.doc(), 1);
+        assert!((scorer.score() - 0.22920431).abs() < 0.001f32);
+        assert_eq!(scorer.advance(), TERMINATED);
+        Ok(())
+    }
+
+    #[test]
+    fn test_fieldnorm_enabled() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let text_options = TextOptions::default()
+            .set_indexing_options(TextFieldIndexing::default().set_fieldnorms(true));
+        let text = schema_builder.add_text_field("text", text_options);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut writer = index.writer_for_tests()?;
+        writer.add_document(doc!(text=>"hello"))?;
+        writer.add_document(doc!(text=>"hello hello hello"))?;
+        writer.commit()?;
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+        let query = TermQuery::new(
+            Term::from_field_text(text, "hello"),
+            IndexRecordOption::WithFreqs,
+        );
+        let weight = query.weight(&*searcher, true)?;
+        let mut scorer = weight.scorer(searcher.segment_reader(0), 1.0f32)?;
+        assert_eq!(scorer.doc(), 0);
+        assert!((scorer.score() - 0.22920431).abs() < 0.001f32);
+        assert_eq!(scorer.advance(), 1);
+        assert_eq!(scorer.doc(), 1);
+        assert!((scorer.score() - 0.15136132).abs() < 0.001f32);
+        assert_eq!(scorer.advance(), TERMINATED);
+        Ok(())
+    }
+}

src/fieldnorm/writer.rs

@@ -4,6 +4,7 @@ use super::fieldnorm_to_id;
 use super::FieldNormsSerializer;
 use crate::schema::Field;
 use crate::schema::Schema;
+use std::cmp::Ordering;
 use std::{io, iter};
 
 /// The `FieldNormsWriter` is in charge of tracking the fieldnorm byte
@@ -12,8 +13,7 @@ use std::{io, iter};
 /// `FieldNormsWriter` stores a Vec<u8> for each tracked field, using a
 /// byte per document per field.
 pub struct FieldNormsWriter {
-    fields: Vec<Field>,
-    fieldnorms_buffer: Vec<Vec<u8>>,
+    fieldnorms_buffers: Vec<Option<Vec<u8>>>,
 }
 
 impl FieldNormsWriter {
@@ -23,7 +23,7 @@ impl FieldNormsWriter {
         schema
             .fields()
             .filter_map(|(field, field_entry)| {
-                if field_entry.is_indexed() {
+                if field_entry.is_indexed() && field_entry.has_fieldnorms() {
                     Some(field)
                 } else {
                     None
@@ -35,25 +35,20 @@ impl FieldNormsWriter {
     /// Initialize with state for tracking the field norm fields
     /// specified in the schema.
     pub fn for_schema(schema: &Schema) -> FieldNormsWriter {
-        let fields = FieldNormsWriter::fields_with_fieldnorm(schema);
-        let max_field = fields
-            .iter()
-            .map(Field::field_id)
-            .max()
-            .map(|max_field_id| max_field_id as usize + 1)
-            .unwrap_or(0);
-        FieldNormsWriter {
-            fields,
-            fieldnorms_buffer: iter::repeat_with(Vec::new)
-                .take(max_field)
-                .collect::<Vec<_>>(),
+        let mut fieldnorms_buffers: Vec<Option<Vec<u8>>> = iter::repeat_with(|| None)
+            .take(schema.num_fields())
+            .collect();
+        for field in FieldNormsWriter::fields_with_fieldnorm(schema) {
+            fieldnorms_buffers[field.field_id() as usize] = Some(Vec::with_capacity(1_000));
         }
+        FieldNormsWriter { fieldnorms_buffers }
     }
 
     /// The memory used inclusive childs
     pub fn mem_usage(&self) -> usize {
-        self.fieldnorms_buffer
+        self.fieldnorms_buffers
             .iter()
+            .flatten()
             .map(|buf| buf.capacity())
             .sum()
     }
@@ -62,8 +57,10 @@ impl FieldNormsWriter {
     ///
     /// Will extend with 0-bytes for documents that have not been seen.
     pub fn fill_up_to_max_doc(&mut self, max_doc: DocId) {
-        for field in self.fields.iter() {
-            self.fieldnorms_buffer[field.field_id() as usize].resize(max_doc as usize, 0u8);
+        for fieldnorms_buffer_opt in self.fieldnorms_buffers.iter_mut() {
+            if let Some(fieldnorms_buffer) = fieldnorms_buffer_opt.as_mut() {
+                fieldnorms_buffer.resize(max_doc as usize, 0u8);
+            }
         }
     }
 
@@ -76,14 +73,24 @@ impl FieldNormsWriter {
     /// * field     - the field being set
     /// * fieldnorm - the number of terms present in document `doc` in field `field`
     pub fn record(&mut self, doc: DocId, field: Field, fieldnorm: u32) {
-        let fieldnorm_buffer: &mut Vec<u8> = &mut self.fieldnorms_buffer[field.field_id() as usize];
-        assert!(
-            fieldnorm_buffer.len() <= doc as usize,
-            "Cannot register a given fieldnorm twice"
-        );
-        // we fill intermediary `DocId` as  having a fieldnorm of 0.
-        fieldnorm_buffer.resize(doc as usize + 1, 0u8);
-        fieldnorm_buffer[doc as usize] = fieldnorm_to_id(fieldnorm);
+        if let Some(fieldnorm_buffer) = self
+            .fieldnorms_buffers
+            .get_mut(field.field_id() as usize)
+            .map(Option::as_mut)
+            .flatten()
+        {
+            match fieldnorm_buffer.len().cmp(&(doc as usize)) {
+                Ordering::Less => {
+                    // we fill intermediary `DocId` as  having a fieldnorm of 0.
+                    fieldnorm_buffer.resize(doc as usize, 0u8);
+                }
+                Ordering::Equal => {}
+                Ordering::Greater => {
+                    panic!("Cannot register a given fieldnorm twice")
+                }
+            }
+            fieldnorm_buffer.push(fieldnorm_to_id(fieldnorm));
+        }
     }
 
     /// Serialize the seen fieldnorm values to the serializer for all fields.
@@ -92,17 +99,22 @@ impl FieldNormsWriter {
         mut fieldnorms_serializer: FieldNormsSerializer,
         doc_id_map: Option<&DocIdMapping>,
     ) -> io::Result<()> {
-        for &field in self.fields.iter() {
-            let fieldnorm_values: &[u8] = &self.fieldnorms_buffer[field.field_id() as usize][..];
+        for (field, fieldnorms_buffer) in self
+            .fieldnorms_buffers
+            .iter()
+            .enumerate()
+            .map(|(field_id, fieldnorms_buffer_opt)| {
+                fieldnorms_buffer_opt.as_ref().map(|fieldnorms_buffer| {
+                    (Field::from_field_id(field_id as u32), fieldnorms_buffer)
+                })
+            })
+            .flatten()
+        {
             if let Some(doc_id_map) = doc_id_map {
-                let mut mapped_fieldnorm_values = vec![];
-                mapped_fieldnorm_values.resize(fieldnorm_values.len(), 0u8);
-                for (new_doc_id, old_doc_id) in doc_id_map.iter_old_doc_ids().enumerate() {
-                    mapped_fieldnorm_values[new_doc_id] = fieldnorm_values[old_doc_id as usize];
-                }
-                fieldnorms_serializer.serialize_field(field, &mapped_fieldnorm_values)?;
+                let remapped_fieldnorm_buffer = doc_id_map.remap(fieldnorms_buffer);
+                fieldnorms_serializer.serialize_field(field, &remapped_fieldnorm_buffer)?;
             } else {
-                fieldnorms_serializer.serialize_field(field, fieldnorm_values)?;
+                fieldnorms_serializer.serialize_field(field, fieldnorms_buffer)?;
             }
         }
         fieldnorms_serializer.close()?;