From 1fa4018113b2c773836ff3b082415b7be9314123 Mon Sep 17 00:00:00 2001 From: Casper Beyer Date: Fri, 21 Aug 2020 20:26:05 +0800 Subject: [PATCH] Add test harness and test-suite This adds a test harness for running the test-suite in Google Chrome along with the test-suite which is brought in as prebuilt modules via a submodule. --- .github/workflows/ci.yml | 6 + .gitmodules | 3 + test.ts | 340 +++++++++++++++++++++++++++++++++++++++ tests | 1 + 4 files changed, 350 insertions(+) create mode 100644 .gitmodules create mode 100644 test.ts create mode 160000 tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7757e20..58b902e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v2 + with: + submodules: true - name: Set up Deno uses: denolib/setup-deno@v2 @@ -19,3 +21,7 @@ jobs: - name: build run: | deno run --allow-all --unstable build.ts + + - name: Test + run: | + deno run --allow-all --unstable test.ts diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..940a095 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tests"] + path = tests + url = git@github.com:khronosproject/wasi-test-suite.git diff --git a/test.ts b/test.ts new file mode 100644 index 0000000..2cf9a84 --- /dev/null +++ b/test.ts @@ -0,0 +1,340 @@ +import { + serve, +} from "https://deno.land/std/http/server.ts"; + +import { + serveFile, +} from "https://deno.land/std/http/file_server.ts"; + +const ignore = [ + "tests/std_env_args_none.wasm", + "tests/std_env_args_some.wasm", + "tests/std_env_vars_none.wasm", + "tests/std_env_vars_some.wasm", + "tests/std_fs_create_dir_absolute.wasm", + "tests/std_fs_create_dir_relative.wasm", + "tests/std_fs_file_create_absolute.wasm", + "tests/std_fs_file_create_relative.wasm", + "tests/std_fs_file_metadata_absolute.wasm", + "tests/std_fs_file_metadata_relative.wasm", + "tests/std_fs_file_seek_absolute.wasm", + "tests/std_fs_file_seek_relative.wasm", + "tests/std_fs_file_set_len_absolute.wasm", + "tests/std_fs_file_set_len_relative.wasm", + "tests/std_fs_file_sync_all_absolute.wasm", + "tests/std_fs_file_sync_all_relative.wasm", + "tests/std_fs_file_sync_data_absolute.wasm", + "tests/std_fs_file_sync_data_relative.wasm", + "tests/std_fs_hard_link_absolute.wasm", + "tests/std_fs_hard_link_relative.wasm", + "tests/std_fs_metadata_absolute.wasm", + "tests/std_fs_metadata_relative.wasm", + "tests/std_fs_read_absolute.wasm", + "tests/std_fs_read_dir_absolute.wasm", + "tests/std_fs_read_dir_relative.wasm", + "tests/std_fs_read_relative.wasm", + "tests/std_fs_remove_dir_all_absolute.wasm", + "tests/std_fs_remove_dir_all_relative.wasm", + "tests/std_fs_rename_absolute.wasm", + "tests/std_fs_rename_relative.wasm", + "tests/std_fs_symlink_metadata_absolute.wasm", + "tests/std_fs_symlink_metadata_relative.wasm", + "tests/std_fs_write_absolute.wasm", + "tests/std_fs_write_relative.wasm", + "tests/std_io_stderr.wasm", + "tests/std_io_stdin.wasm", + "tests/std_io_stdout.wasm", + "tests/std_process_exit.wasm", + "tests/wasi_clock_res_get_monotonic.wasm", + "tests/wasi_clock_res_get_process.wasm", + "tests/wasi_clock_res_get_realtime.wasm", + "tests/wasi_clock_res_get_thread.wasm", + "tests/wasi_clock_time_get_monotonic.wasm", + "tests/wasi_clock_time_get_process.wasm", + "tests/wasi_clock_time_get_realtime.wasm", + "tests/wasi_clock_time_get_thread.wasm", + "tests/wasi_fd_write_file.wasm", + "tests/wasi_fd_write_stderr.wasm", + "tests/wasi_fd_write_stdout.wasm", + "tests/wasi_proc_exit_one.wasm", + "tests/wasi_proc_exit_zero.wasm", + "tests/wasi_random_get.wasm", +]; + +const manifest: { [key: string]: unknown } = {}; +for await (const entry of Deno.readDir("tests")) { + if (!entry.name.endsWith(".wasm")) { + continue; + } + + const name = `tests/${entry.name}`; + const path = name.replace(/\.wasm$/, ".json"); + manifest[name] = JSON.parse(await Deno.readTextFile(path)); +} + +const server = serve({ port: 8080 }); + +const browser = await Deno.run({ + cmd: [ + chromePath(), + ...chromeFlags(), + "http://localhost:8080", + ], + stdout: "null", + stderr: "null", +}); + +const result: { + passed: number; + failed: number; + ignored: number; + errors: string[]; +} = { + passed: 0, + failed: 0, + ignored: 0, + errors: [], +}; + +for await (const request of server) { + try { + switch (request.url) { + case "/": + case "/index.html": { + await request.respond(await serveIndex()); + break; + } + + case "/runner.js": { + await request.respond(await serveRunner(manifest, ignore)); + break; + } + + case "/favicon.ico": { + await request.respond({ body: "" }); + break; + } + + case "/test": { + await request.respond({ body: "" }); + const body = await Deno.readAll(request.body); + + await Deno.writeAll(Deno.stderr, new TextEncoder().encode("test ")); + await Deno.writeAll(Deno.stderr, body); + await Deno.writeAll(Deno.stderr, new TextEncoder().encode("...")); + break; + } + + case "/pass": { + await request.respond({ body: "" }); + result.passed++; + + await Deno.writeAll(Deno.stderr, new TextEncoder().encode("ok\n")); + break; + } + + case "/fail": { + await request.respond({ body: "" }); + const body = await Deno.readAll(request.body); + + result.errors.push(new TextDecoder().decode(body)); + result.failed++; + + await Deno.writeAll(Deno.stderr, new TextEncoder().encode("FAILED\n")); + break; + } + + case "/skip": { + await request.respond({ body: "" }); + const body = await Deno.readAll(request.body); + + result.ignored++; + + await Deno.writeAll(Deno.stderr, new TextEncoder().encode("ignore\n")); + break; + } + + default: { + const filepath = request.url.slice(1); + await request.respond(await serveFile(request, filepath)); + break; + } + } + } catch (error) { + console.error(request.url, error); + } + + const pending = Object.keys(manifest).length - + result.passed - + result.failed - + result.ignored; + + if (pending == 0) { + break; + } +} + +browser.close(); +server.close(); + +if (result.errors.length > 0) { + await Deno.writeAll( + Deno.stderr, + new TextEncoder().encode(`\n`), + ); + + for (const error of result.errors) { + await Deno.writeAll( + Deno.stderr, + new TextEncoder().encode(`${error}\n`), + ); + } +} + +await Deno.writeAll( + Deno.stderr, + new TextEncoder().encode( + `\ntest results: ${ + result.failed ? "FAILED" : "ok" + }. ${result.passed} passed; ${result.failed} failed; ignored; ${result.ignored}\n`, + ), +); + +if (result.failed) { + Deno.exit(1); +} + +async function serveIndex() { + const body = ` + + + + + + + + + + `; + + return { + body, + }; +} + +async function serveRunner(manifest: unknown, ignore: string[]) { + const headers = new Headers({ + "Content-Type": "application/javascript", + }); + + const body = ` + import Context from "/lib/wasi_snapshot_preview1.js"; + + const manifest = ${JSON.stringify(manifest)}; + const ignore = ${JSON.stringify(ignore)}; + + async function test(name) { + return fetch("http://localhost:8080/test", { + method: 'POST', + body: name, + }); + } + + async function pass() { + return fetch("http://localhost:8080/pass", { + method: 'POST', + }); + } + + async function fail(error) { + return fetch("http://localhost:8080/fail", { + method: 'POST', + body: error, + }); + } + + async function skip(error) { + return fetch("http://localhost:8080/skip", { + method: 'POST', + body: error, + }); + } + + window.onerror = async function() { + write("error"); + }; + + window.onload = async function() { + const result = { + passed: 0, + failed: 0, + errors: [], + }; + + const entries = Object.entries(manifest).sort(); + for (const [pathname, options] of entries) { + await test(pathname); + + if (ignore.includes(pathname)) { + await skip(); + continue; + } + + try { + const context = new Context({ + }); + + const request = await fetch(pathname); + const binary = await request.arrayBuffer(); + const module = await WebAssembly.compile(binary); + const instance = await WebAssembly.instantiate(module, { + wasi_snapshot_preview1: context.exports, + }); + + instance.exports._start(); + await pass(); + } catch (error) { + await fail(error.stack); + } + } + };`; + + return { + headers, + body, + }; +} + +function chromePath() { + switch (Deno.build.os) { + case "darwin": + return "/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome"; + + case "linux": + return "/usr/bin/google-chrome"; + + case "windows": + return "C:\Program Files (x86)\Google\Chrome\Application\chrome.exe"; + } +} + +function chromeFlags(): string[] { + return [ + "--headless", + "--remote-debugging-port=9292", + "--disable-features=TranslateUI", + "--disable-extensions", + "--disable-component-extensions-with-background-pages", + "--disable-background-networking", + "--disable-sync", + "--metrics-recording-only", + "--disable-default-apps", + "--mute-audio", + "--no-default-browser-check", + "--no-first-run", + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", + "--disable-background-timer-throttling", + "--force-fieldtrials=*BackgroundTracing/default/", + ]; +} diff --git a/tests b/tests new file mode 160000 index 0000000..b32e8a1 --- /dev/null +++ b/tests @@ -0,0 +1 @@ +Subproject commit b32e8a128a75db1e269059a96e90308c33a365d2