Datagen API improvements (#3705)

unicode-org · Jul 20, 2023 · c2064d9 · c2064d9
1 parent f000c35
commit c2064d9
Show file tree

Hide file tree

Showing 29 changed files with 295 additions and 244 deletions.
diff --git a/Makefile.toml b/Makefile.toml
@@ -91,6 +91,7 @@ description = "Run all tests for the CI 'testdata' job"
 category = "CI"
 dependencies = [
     "testdata-check",
+    "testdata-legacy",
 ]
 
 [tasks.ci-job-test-docs]

diff --git a/components/calendar/data/config.json b/components/calendar/data/config.json
@@ -12,8 +12,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/components/collator/data/config.json b/components/collator/data/config.json
@@ -16,8 +16,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/components/datetime/data/config.json b/components/datetime/data/config.json
@@ -33,8 +33,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/components/decimal/data/config.json b/components/decimal/data/config.json
@@ -10,8 +10,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/components/list/data/config.json b/components/list/data/config.json
@@ -13,8 +13,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/components/locid_transform/data/config.json b/components/locid_transform/data/config.json
@@ -17,8 +17,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/components/normalizer/data/config.json b/components/normalizer/data/config.json
@@ -16,8 +16,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/components/plurals/data/config.json b/components/plurals/data/config.json
@@ -11,8 +11,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/components/properties/data/config.json b/components/properties/data/config.json
@@ -119,8 +119,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/components/segmenter/data/config.json b/components/segmenter/data/config.json
@@ -16,8 +16,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/components/timezone/data/config.json b/components/timezone/data/config.json
@@ -10,8 +10,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/docs/tutorials/crates/baked/build.rs b/docs/tutorials/crates/baked/build.rs
@@ -14,6 +14,7 @@ fn main() {
     let mut options = BakedOptions::default();
     // Overwrite the baked data if it was already present:
     options.overwrite = true;
+    options.use_separate_crates = false;
 
 
     icu_datagen::datagen(

diff --git a/experimental/casemap/data/config.json b/experimental/casemap/data/config.json
@@ -10,8 +10,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/experimental/compactdecimal/data/config.json b/experimental/compactdecimal/data/config.json
@@ -12,8 +12,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/experimental/displaynames/data/config.json b/experimental/displaynames/data/config.json
@@ -14,8 +14,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/experimental/relativetime/data/config.json b/experimental/relativetime/data/config.json
@@ -33,8 +33,7 @@
     "segmenter_lstm": "Latest",
     "export": {
         "Baked": {
-            "output_path": "data",
-            "use_separate_crates": true,
+            "path": "data",
             "pretty": true
         }
     },

diff --git a/provider/adapters/tests/data/config.json b/provider/adapters/tests/data/config.json
@@ -9,10 +9,12 @@
     },
     "locales": "All",
     "cldr": "Latest",
-    "icu_export": "Latest",
-    "segmenter_lstm": "Latest",
+    "icu_export": "None",
+    "segmenter_lstm": "None",
     "export": {
-        "Blob": "blob.postcard"
+        "Blob": {
+            "path": "blob.postcard"
+        }
     },
     "fallback": "Legacy",
     "overwrite": true

diff --git a/provider/adapters/tests/data/langtest/de.json b/provider/adapters/tests/data/langtest/de.json
@@ -0,0 +1,22 @@
+{
+    "keys": {
+      "Explicit": [
+        "core/helloworld@1"
+      ]
+    },
+    "locales": {
+      "Explicit": [
+        "de"
+      ]
+    },
+    "cldr": "None",
+    "icu_export": "None",
+    "segmenter_lstm": "None",
+    "export": {
+      "Fs": {
+        "path": "de",
+        "syntax": "Json"
+      }
+    },
+    "overwrite": true
+  }
diff --git a/provider/adapters/tests/data/langtest/ro.json b/provider/adapters/tests/data/langtest/ro.json
@@ -0,0 +1,22 @@
+{
+    "keys": {
+      "Explicit": [
+        "core/helloworld@1"
+      ]
+    },
+    "locales": {
+      "Explicit": [
+        "ro"
+      ]
+    },
+    "cldr": "None",
+    "icu_export": "None",
+    "segmenter_lstm": "None",
+    "export": {
+      "Fs": {
+        "path": "ro",
+        "syntax": "Json"
+      }
+    },
+    "overwrite": true
+  }
diff --git a/provider/blob/tests/data/config.json b/provider/blob/tests/data/config.json
@@ -0,0 +1,16 @@
+{
+  "keys": {
+    "Explicit": [
+      "core/helloworld@1"
+    ]
+  },
+  "cldr": "None",
+  "icu_export": "None",
+  "segmenter_lstm": "None",
+  "export": {
+    "Blob": {
+      "path": "hello_world.postcard"
+    }
+  },
+  "overwrite": true
+}
diff --git a/provider/datagen/src/baked_exporter.rs b/provider/datagen/src/baked_exporter.rs
@@ -112,7 +112,7 @@ impl Default for Options {
         Self {
             pretty: false,
             insert_feature_gates: false,
-            use_separate_crates: false,
+            use_separate_crates: true,
             overwrite: false,
         }
     }

diff --git a/provider/datagen/src/bin/datagen/args.rs b/provider/datagen/src/bin/datagen/args.rs
@@ -240,8 +240,45 @@ pub struct Cli {
 
 impl Cli {
     pub fn as_config(&self) -> eyre::Result<config::Config> {
-        Ok(if let Some(ref path) = self.config {
-            serde_json::from_str(&std::fs::read_to_string(path)?)?
+        Ok(if let Some(ref config_path) = self.config {
+            let mut config: config::Config =
+                serde_json::from_str(&std::fs::read_to_string(config_path)?)?;
+            let parent = config_path.parent().unwrap();
+            // all paths in the JSON file are relative to its path, not to pwd.
+            for path_or_tag in [
+                &mut config.cldr,
+                &mut config.icu_export,
+                &mut config.segmenter_lstm,
+            ] {
+                if let config::PathOrTag::Path(ref mut path) = path_or_tag {
+                    if path.is_relative() {
+                        *path = parent.join(path.clone());
+                    }
+                }
+            }
+            if let config::KeyInclude::ForBinary(path) = &mut config.keys {
+                if path.is_relative() {
+                    *path = parent.join(path.clone());
+                }
+            }
+            match &mut config.export {
+                config::Export::Fs { path, .. } => {
+                    if path.is_relative() {
+                        *path = parent.join(path.clone());
+                    }
+                }
+                config::Export::Blob { path, .. } => {
+                    if path.is_relative() {
+                        *path = parent.join(path.clone());
+                    }
+                }
+                config::Export::Baked { path, .. } => {
+                    if path.is_relative() {
+                        *path = parent.join(path.clone());
+                    }
+                }
+            }
+            config
         } else {
             config::Config {
                 keys: self.make_keys()?,
@@ -378,7 +415,7 @@ impl Cli {
                 eyre::bail!("FsDataProvider export requires the provider_fs Cargo feature.");
                 #[cfg(feature = "provider_fs")]
                 Ok(config::Export::Fs {
-                    output_path: if let Some(root) = self.output.as_ref() {
+                    path: if let Some(root) = self.output.as_ref() {
                         root.clone()
                     } else {
                         PathBuf::from("icu4x_data")
@@ -396,25 +433,27 @@ impl Cli {
                 #[cfg(not(feature = "provider_blob"))]
                 eyre::bail!("BlobDataProvider export requires the provider_blob Cargo feature.");
                 #[cfg(feature = "provider_blob")]
-                Ok(config::Export::Blob(if let Some(path) = &self.output {
-                    path.clone()
-                } else {
-                    PathBuf::from("/stdout")
-                }))
+                Ok(config::Export::Blob {
+                    path: if let Some(path) = &self.output {
+                        path.clone()
+                    } else {
+                        PathBuf::from("/stdout")
+                    },
+                })
             }
             Format::Mod => {
                 #[cfg(not(feature = "provider_baked"))]
                 eyre::bail!("Baked data export requires the provider_baked Cargo feature.");
                 #[cfg(feature = "provider_baked")]
                 Ok(config::Export::Baked {
-                    output_path: if let Some(mod_directory) = self.output.as_ref() {
+                    path: if let Some(mod_directory) = self.output.as_ref() {
                         mod_directory.clone()
                     } else {
                         PathBuf::from("icu4x_data")
                     },
                     pretty: self.pretty,
                     insert_feature_gates: self.insert_feature_gates,
-                    use_separate_crates: self.use_separate_crates,
+                    use_meta_crate: !self.use_separate_crates,
                 })
             }
         }