chore: rename table metadata to config and merge SetConfig and DeleteConfig

Author: dsgibbons

protos/table.proto

@@ -86,7 +86,7 @@ message Manifest {
   // * 2: move_stable_row_ids: row IDs are tracked and stable after move operations
   //       (such as compaction), but not updates.
   // * 4: use v2 format (deprecated)
-  // * 8: table metadata are present
+  // * 8: table config is present
   uint64 reader_feature_flags = 9;
 
   // Feature flags for writers.
@@ -140,12 +140,12 @@ message Manifest {
   // This specifies what format is used to store the data files.
   DataStorageFormat data_format = 15;
 
-  // Table metadata.
+  // Table config.
   //
   // Keys with the prefix "lance." are reserved for the Lance library. Other
   // libraries may wish to similarly prefix their configuration keys
   // appropriately.
-  map<string, string> table_metadata = 16;
+  map<string, string> config = 16;
 } // Manifest
 
 // Auxiliary Data attached to a version.

protos/transaction.proto

@@ -61,8 +61,8 @@ message Transaction {
     repeated lance.file.Field schema = 2;
     // Schema metadata.
     map<string, bytes> schema_metadata = 3;
-    // Table metadata to merge with existing table metadata.
-    map<string, string> table_metadata = 4;
+    // Key-value pairs to merge with existing config.
+    map<string, string> config_upsert_values = 4;
   }
 
   // Add or replace a new secondary index.
@@ -159,16 +159,11 @@ message Transaction {
     repeated DataFragment new_fragments = 3;
   }
 
-  // An operation that sets table metadata.
-  message SetMetadata {
-    map<string, string> metadata = 1;
+  // An operation that updates the table config.
+  message UpdateConfig {
+    map<string, string> upsert_values = 1;
+    repeated string delete_keys = 2;
   }
-  
-  // An operation that deletes table metadata.
-  message DeleteMetadata {
-    repeated string metadata_keys = 1;
-  }
-
 
   // The operation of this transaction.
   oneof operation {
@@ -182,7 +177,6 @@ message Transaction {
     ReserveFragments reserve_fragments = 107;
     Update update = 108;
     Project project = 109;
-    SetMetadata set_metadata = 110;
-    DeleteMetadata delete_metadata = 111;
+    UpdateConfig update_config = 110;
   }
 }

python/src/dataset.rs

@@ -272,7 +272,7 @@ impl Operation {
         let op = LanceOperation::Overwrite {
             fragments,
             schema,
-            table_metadata: None,
+            config_upsert_values: None,
         };
         Ok(Self(op))
     }

rust/lance-table/src/feature_flags.rs

@@ -58,7 +58,7 @@ pub fn apply_feature_flags(manifest: &mut Manifest, enable_stable_row_id: bool)
     }
 
     // Test whether any table metadata has been set
-    if !manifest.table_metadata.is_empty() {
+    if !manifest.config.is_empty() {
         manifest.reader_feature_flags |= FLAG_TABLE_METADATA;
         manifest.writer_feature_flags |= FLAG_TABLE_METADATA;
     }

rust/lance-table/src/format/manifest.rs

@@ -79,8 +79,8 @@ pub struct Manifest {
     /// The storage format of the data files.
     pub data_storage_format: DataStorageFormat,
 
-    /// Table metadata.
-    pub table_metadata: HashMap<String, String>,
+    /// Table configuration.
+    pub config: HashMap<String, String>,
 }
 
 fn compute_fragment_offsets(fragments: &[Fragment]) -> Vec<usize> {
@@ -120,7 +120,7 @@ impl Manifest {
             fragment_offsets,
             next_row_id: 0,
             data_storage_format,
-            table_metadata: HashMap::new(),
+            config: HashMap::new(),
         }
     }
 
@@ -147,7 +147,7 @@ impl Manifest {
             fragment_offsets,
             next_row_id: previous.next_row_id,
             data_storage_format: previous.data_storage_format.clone(),
-            table_metadata: previous.table_metadata.clone(),
+            config: previous.config.clone(),
         }
     }
 
@@ -167,15 +167,15 @@ impl Manifest {
         self.timestamp_nanos = nanos;
     }
 
-    /// Set the `table_metadata` from a metadata iterator
-    pub fn set_metadata(&mut self, metadata: impl IntoIterator<Item = (String, String)>) {
-        self.table_metadata.extend(metadata);
+    /// Set the `config` from an iterator
+    pub fn update_config(&mut self, upsert_values: impl IntoIterator<Item = (String, String)>) {
+        self.config.extend(upsert_values);
     }
 
-    /// Delete `table_metadata` keys using a slice of metadata keys
-    pub fn delete_metadata(&mut self, metadata_keys: &[&str]) {
-        self.table_metadata
-            .retain(|key, _| !metadata_keys.contains(&key.as_str()));
+    /// Delete `config` keys using a slice of keys
+    pub fn delete_config_keys(&mut self, delete_keys: &[&str]) {
+        self.config
+            .retain(|key, _| !delete_keys.contains(&key.as_str()));
     }
 
     /// Check the current fragment list and update the high water mark
@@ -489,7 +489,7 @@ impl TryFrom<pb::Manifest> for Manifest {
             fragment_offsets,
             next_row_id: p.next_row_id,
             data_storage_format,
-            table_metadata: p.table_metadata,
+            config: p.config,
         })
     }
 }
@@ -532,7 +532,7 @@ impl From<&Manifest> for pb::Manifest {
                 file_format: m.data_storage_format.file_format.clone(),
                 version: m.data_storage_format.version.clone(),
             }),
-            table_metadata: m.table_metadata.clone(),
+            config: m.config.clone(),
         }
     }
 }
@@ -714,7 +714,7 @@ mod tests {
     }
 
     #[test]
-    fn test_table_metadata() {
+    fn test_config() {
         let arrow_schema = ArrowSchema::new(vec![ArrowField::new(
             "a",
             arrow_schema::DataType::Int64,
@@ -728,15 +728,15 @@ mod tests {
         ];
         let mut manifest = Manifest::new(schema, Arc::new(fragments), DataStorageFormat::default());
 
-        let mut metadata = HashMap::new();
-        metadata.insert("lance:test".to_string(), "value".to_string());
-        metadata.insert("other-key".to_string(), "other-value".to_string());
+        let mut config = HashMap::new();
+        config.insert("lance:test".to_string(), "value".to_string());
+        config.insert("other-key".to_string(), "other-value".to_string());
 
-        manifest.set_metadata(metadata.clone());
-        assert_eq!(manifest.table_metadata, metadata.clone());
+        manifest.update_config(config.clone());
+        assert_eq!(manifest.config, config.clone());
 
-        metadata.remove("other-key");
-        manifest.delete_metadata(&["other-key"]);
-        assert_eq!(manifest.table_metadata, metadata);
+        config.remove("other-key");
+        manifest.delete_config_keys(&["other-key"]);
+        assert_eq!(manifest.config, config);
     }
 }

rust/lance-table/src/io/manifest.rs

@@ -244,8 +244,8 @@ mod test {
             ArrowSchema::new(vec![ArrowField::new(long_name, DataType::Int64, false)]);
         let schema = Schema::try_from(&arrow_schema).unwrap();
 
-        let mut table_metadata = HashMap::new();
-        table_metadata.insert("key".to_string(), "value".to_string());
+        let mut config = HashMap::new();
+        config.insert("key".to_string(), "value".to_string());
 
         let mut manifest = Manifest::new(schema, Arc::new(vec![]), DataStorageFormat::default());
         let pos = write_manifest(&mut writer, &mut manifest, None)

rust/lance/src/dataset.rs

@@ -593,7 +593,7 @@ impl Dataset {
             WriteMode::Create | WriteMode::Overwrite => Operation::Overwrite {
                 schema,
                 fragments,
-                table_metadata: None,
+                config_upsert_values: None,
             },
             WriteMode::Append => Operation::Append { fragments },
         };
@@ -1510,15 +1510,16 @@ impl Dataset {
         self.merge_impl(stream, left_on, right_on).await
     }
 
-    /// Set key-value pairs in table metadata.
-    pub async fn set_table_metadata(
+    /// Update key-value pairs in config.
+    pub async fn update_config(
         &mut self,
-        metadata: impl IntoIterator<Item = (String, String)>,
+        upsert_values: impl IntoIterator<Item = (String, String)>,
     ) -> Result<()> {
         let transaction = Transaction::new(
             self.manifest.version,
-            Operation::SetMetadata {
-                table_metadata: HashMap::from_iter(metadata),
+            Operation::UpdateConfig {
+                upsert_values: Some(HashMap::from_iter(upsert_values)),
+                delete_keys: None,
             },
             None,
         );
@@ -1539,12 +1540,13 @@ impl Dataset {
         Ok(())
     }
 
-    /// Delete keys from the table metadata.
-    pub async fn delete_table_metadata(&mut self, metadata_keys: &[&str]) -> Result<()> {
+    /// Delete keys from the config.
+    pub async fn delete_config_keys(&mut self, delete_keys: &[&str]) -> Result<()> {
         let transaction = Transaction::new(
             self.manifest.version,
-            Operation::DeleteMetadata {
-                table_metadata_keys: Vec::from_iter(metadata_keys.iter().map(ToString::to_string)),
+            Operation::UpdateConfig {
+                upsert_values: None,
+                delete_keys: Some(Vec::from_iter(delete_keys.iter().map(ToString::to_string))),
             },
             None,
         );
@@ -2861,7 +2863,7 @@ mod tests {
         let operation = Operation::Overwrite {
             fragments: vec![],
             schema,
-            table_metadata: None,
+            config_upsert_values: None,
         };
         let test_dir = tempdir().unwrap();
         let test_uri = test_dir.path().to_str().unwrap();
@@ -3299,7 +3301,7 @@ mod tests {
 
     #[rstest]
     #[tokio::test]
-    async fn test_table_metadata() {
+    async fn test_update_config() {
         // Create a table
         let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
             "i",
@@ -3317,16 +3319,16 @@ mod tests {
         let reader = RecordBatchIterator::new(vec![data.unwrap()].into_iter().map(Ok), schema);
         let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
 
-        let mut metadata = HashMap::new();
-        metadata.insert("lance:test".to_string(), "value".to_string());
-        metadata.insert("other-key".to_string(), "other-value".to_string());
+        let mut desired_config = HashMap::new();
+        desired_config.insert("lance:test".to_string(), "value".to_string());
+        desired_config.insert("other-key".to_string(), "other-value".to_string());
 
-        dataset.set_table_metadata(metadata.clone()).await.unwrap();
-        assert_eq!(dataset.manifest.table_metadata, metadata);
+        dataset.update_config(desired_config.clone()).await.unwrap();
+        assert_eq!(dataset.manifest.config, desired_config);
 
-        metadata.remove("other-key");
-        dataset.delete_table_metadata(&["other-key"]).await.unwrap();
-        assert_eq!(dataset.manifest.table_metadata, metadata);
+        desired_config.remove("other-key");
+        dataset.delete_config_keys(&["other-key"]).await.unwrap();
+        assert_eq!(dataset.manifest.config, desired_config);
     }
 
     #[rstest]

rust/lance/src/dataset/fragment.rs

@@ -2357,7 +2357,7 @@ mod tests {
         let op = Operation::Overwrite {
             schema: schema.clone(),
             fragments,
-            table_metadata: None,
+            config_upsert_values: None,
         };
 
         let registry = Arc::new(ObjectStoreRegistry::default());
@@ -2458,7 +2458,7 @@ mod tests {
             let op = Operation::Overwrite {
                 fragments: vec![new_fragment],
                 schema: full_schema.clone(),
-                table_metadata: None,
+                config_upsert_values: None,
             };
 
             let registry = Arc::new(ObjectStoreRegistry::default());

rust/lance/src/dataset/transaction.rs

@@ -939,14 +939,14 @@ mod tests {
     async fn test_good_concurrent_config_writes() {
         let dataset = get_empty_dataset().await;
 
-        // Test successful concurrent set operations
+        // Test successful concurrent insert config operations
         let futures: Vec<_> = ["key1", "key2", "key3", "key4", "key5"]
             .iter()
             .map(|key| {
                 let mut dataset = dataset.clone();
                 tokio::spawn(async move {
                     dataset
-                        .set_table_metadata(vec![(key.to_string(), "value".to_string())])
+                        .update_config(vec![(key.to_string(), "value".to_string())])
                         .await
                 })
             })
@@ -959,7 +959,7 @@ mod tests {
         }
 
         let dataset = dataset.checkout_version(6).await.unwrap();
-        assert_eq!(dataset.manifest.table_metadata.len(), 5);
+        assert_eq!(dataset.manifest.config.len(), 5);
 
         dataset.validate().await.unwrap();
 
@@ -969,7 +969,7 @@ mod tests {
             .iter()
             .map(|key| {
                 let mut dataset = dataset.clone();
-                tokio::spawn(async move { dataset.delete_table_metadata(&[key]).await })
+                tokio::spawn(async move { dataset.delete_config_keys(&[key]).await })
             })
             .collect();
         let results = join_all(futures).await;
@@ -982,14 +982,14 @@ mod tests {
         let dataset = dataset.checkout_version(11).await.unwrap();
 
         // There are now two fewer keys
-        assert_eq!(dataset.manifest.table_metadata.len(), 3);
+        assert_eq!(dataset.manifest.config.len(), 3);
 
         dataset.validate().await.unwrap()
     }
 
     #[tokio::test]
     async fn test_bad_concurrent_config_writes() {
-        // If two concurrent set metadata operations occur for the same key, a
+        // If two concurrent insert config operations occur for the same key, a
         // `CommitConflict` should be returned
         let dataset = get_empty_dataset().await;
 
@@ -999,7 +999,7 @@ mod tests {
                 let mut dataset = dataset.clone();
                 tokio::spawn(async move {
                     dataset
-                        .set_table_metadata(vec![(key.to_string(), "value".to_string())])
+                        .update_config(vec![(key.to_string(), "value".to_string())])
                         .await
                 })
             })

rust/lance/src/io/commit.rs

@@ -113,7 +113,7 @@ impl TestDatasetGenerator {
         let operation = Operation::Overwrite {
             fragments,
             schema,
-            table_metadata: None,
+            config_upsert_values: None,
         };
 
         let registry = Arc::new(ObjectStoreRegistry::default());