From 9b8047e6be5e8ef1cfe1439a8bdb1f922a96c140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sat, 21 May 2022 02:12:40 +0200 Subject: [PATCH 1/3] run examples in wasm in CI --- .github/start-wasm-example/.gitignore | 4 + .github/start-wasm-example/package-lock.json | 60 ++++++++++ .github/start-wasm-example/package.json | 13 +++ .../start-wasm-example/playwright.config.ts | 108 ++++++++++++++++++ .../tests/wasm_example.spec.ts | 48 ++++++++ .github/workflows/validation-jobs.yml | 65 +++++++++++ crates/bevy_app/src/ci_testing.rs | 25 +++- 7 files changed, 317 insertions(+), 6 deletions(-) create mode 100644 .github/start-wasm-example/.gitignore create mode 100644 .github/start-wasm-example/package-lock.json create mode 100644 .github/start-wasm-example/package.json create mode 100644 .github/start-wasm-example/playwright.config.ts create mode 100644 .github/start-wasm-example/tests/wasm_example.spec.ts create mode 100644 .github/workflows/validation-jobs.yml diff --git a/.github/start-wasm-example/.gitignore b/.github/start-wasm-example/.gitignore new file mode 100644 index 0000000000000..75e854d8dcf7a --- /dev/null +++ b/.github/start-wasm-example/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +/test-results/ +/playwright-report/ +/playwright/.cache/ diff --git a/.github/start-wasm-example/package-lock.json b/.github/start-wasm-example/package-lock.json new file mode 100644 index 0000000000000..f27430f74fda1 --- /dev/null +++ b/.github/start-wasm-example/package-lock.json @@ -0,0 +1,60 @@ +{ + "name": "start-wasm-example", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "start-wasm-example", + "version": "1.0.0", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.22.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.22.1.tgz", + "integrity": "sha512-8ouMBUboYslHom41W8bnSEn0TwlAMHhCACwOZeuiAgzukj7KobpZ+UBwrGE0jJ0UblJbKAQNRHXL+z7sDSkb6g==", + "dev": true, + "dependencies": { + "playwright-core": "1.22.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/playwright-core": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.22.1.tgz", + "integrity": "sha512-H+ZUVYnceWNXrRf3oxTEKAr81QzFsCKu5Fp//fEjQvqgKkfA1iX3E9DBrPJpPNOrgVzcE+IqeI0fDmYJe6Ynnw==", + "dev": true, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=14" + } + } + }, + "dependencies": { + "@playwright/test": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.22.1.tgz", + "integrity": "sha512-8ouMBUboYslHom41W8bnSEn0TwlAMHhCACwOZeuiAgzukj7KobpZ+UBwrGE0jJ0UblJbKAQNRHXL+z7sDSkb6g==", + "dev": true, + "requires": { + "playwright-core": "1.22.1" + } + }, + "playwright-core": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.22.1.tgz", + "integrity": "sha512-H+ZUVYnceWNXrRf3oxTEKAr81QzFsCKu5Fp//fEjQvqgKkfA1iX3E9DBrPJpPNOrgVzcE+IqeI0fDmYJe6Ynnw==", + "dev": true + } + } +} diff --git a/.github/start-wasm-example/package.json b/.github/start-wasm-example/package.json new file mode 100644 index 0000000000000..236c3251b4fec --- /dev/null +++ b/.github/start-wasm-example/package.json @@ -0,0 +1,13 @@ +{ + "name": "start-wasm-example", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": {}, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.22.1" + } +} diff --git a/.github/start-wasm-example/playwright.config.ts b/.github/start-wasm-example/playwright.config.ts new file mode 100644 index 0000000000000..e9d67bef1c141 --- /dev/null +++ b/.github/start-wasm-example/playwright.config.ts @@ -0,0 +1,108 @@ +import type { PlaywrightTestConfig } from '@playwright/test'; +import { devices } from '@playwright/test'; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// require('dotenv').config(); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +const config: PlaywrightTestConfig = { + testDir: './tests', + /* Maximum time one test can run for. */ + timeout: 600 * 1000, + expect: { + /** + * Maximum time expect() should wait for the condition to be met. + * For example in `await expect(locator).toHaveText();` + */ + timeout: 5000 + }, + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: 0, + /* Opt out of parallel tests on CI. */ + workers: 1, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'list', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ + actionTimeout: 0, + /* Base URL to use in actions like `await page.goto('/')`. */ + // baseURL: 'http://localhost:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { + ...devices['Desktop Chrome'], + }, + }, + + { + name: 'firefox', + use: { + ...devices['Desktop Firefox'], + }, + }, + + // Webkit doesn't work + // { + // name: 'webkit', + // use: { + // ...devices['Desktop Safari'], + // }, + // }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { + // ...devices['Pixel 5'], + // }, + // }, + // { + // name: 'Mobile Safari', + // use: { + // ...devices['iPhone 12'], + // }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { + // channel: 'msedge', + // }, + // }, + // { + // name: 'Google Chrome', + // use: { + // channel: 'chrome', + // }, + // }, + ], + + /* Folder for test artifacts such as screenshots, videos, traces, etc. */ + // outputDir: 'test-results/', + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // port: 3000, + // }, +}; + +export default config; diff --git a/.github/start-wasm-example/tests/wasm_example.spec.ts b/.github/start-wasm-example/tests/wasm_example.spec.ts new file mode 100644 index 0000000000000..023e7e17de057 --- /dev/null +++ b/.github/start-wasm-example/tests/wasm_example.spec.ts @@ -0,0 +1,48 @@ +import { test, expect, Page } from '@playwright/test'; + +test.beforeEach(async ({ page }) => { + await page.goto('http://localhost:8000/'); +}); + +test.describe('WASM example', () => { + test('Wait for success', async ({ page }, test_info) => { + let start = new Date().getTime(); + + let found = false; + while (new Date().getTime() - start < 600000) { + let msg = await promise_with_timeout(100, on_console(page), "no log found"); + if (msg.includes("no log found")) { + continue; + } + console.log(msg); + if (msg.includes("Test successful")) { + await page.screenshot({ path: `screenshot-${test_info.project.name}.png`, fullPage: true }); + found = true; + break; + } + } + + expect(found).toBe(true); + }); + +}); + +function on_console(page) { + return new Promise(resolve => { + page.on('console', msg => resolve(msg.text())); + }); +} + +async function promise_with_timeout(time_limit, task, failure_value) { + let timeout; + const timeout_promise = new Promise((resolve, reject) => { + timeout = setTimeout(() => { + resolve(failure_value); + }, time_limit); + }); + const response = await Promise.race([task, timeout_promise]); + if (timeout) { + clearTimeout(timeout); + } + return response; +} \ No newline at end of file diff --git a/.github/workflows/validation-jobs.yml b/.github/workflows/validation-jobs.yml new file mode 100644 index 0000000000000..b61f2a37ac687 --- /dev/null +++ b/.github/workflows/validation-jobs.yml @@ -0,0 +1,65 @@ +name: validation jobs + +on: + push + +env: + CARGO_TERM_COLOR: always + +jobs: + run-examples-on-wasm: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v3 + + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + target: wasm32-unknown-unknown + override: true + + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-wasm-run-examples-${{ hashFiles('**/Cargo.toml') }} + + - name: install xvfb, llvmpipe and lavapipe + run: | + sudo apt-get update -y -qq + sudo add-apt-repository ppa:oibaf/graphics-drivers -y + sudo apt-get update + sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers + + - name: Install wasm-bindgen + run: cargo install wasm-bindgen-cli + + - name: Run examples + shell: bash + run: | + # start a webserver + python3 -m http.server --directory examples/wasm & + + # build the example in wasm + example=.github/example-run/scene.ron + # example_name=`basename $example .ron` + example_name=lighting + echo "running $example_name - "`date` + cp $example ci_testing_config.ron + cargo build --example $example_name --features "bevy_ci_testing" --target wasm32-unknown-unknown + wasm-bindgen --out-dir examples/wasm/target --out-name wasm_example --target web target/wasm32-unknown-unknown/debug/examples/$example_name.wasm + cd .github/start-wasm-example + npm install + npx playwright install --with-deps + xvfb-run npx playwright test --headed + + - name: Save screenshots + uses: actions/upload-artifact@v3 + with: + name: screenshots + path: .github/start-wasm-example/screenshot-*.png diff --git a/crates/bevy_app/src/ci_testing.rs b/crates/bevy_app/src/ci_testing.rs index b319f04a91750..66206d91095c3 100644 --- a/crates/bevy_app/src/ci_testing.rs +++ b/crates/bevy_app/src/ci_testing.rs @@ -1,6 +1,8 @@ use crate::{app::AppExit, App}; use serde::Deserialize; +use bevy_utils::tracing::info; + /// A configuration struct for automated CI testing. /// /// It gets used when the `bevy_ci_testing` feature is enabled to automatically @@ -20,18 +22,29 @@ fn ci_testing_exit_after( if let Some(exit_after) = ci_testing_config.exit_after { if *current_frame > exit_after { app_exit_events.send(AppExit); + info!("Exiting after {} frames. Test successful!", exit_after); } } *current_frame += 1; } pub(crate) fn setup_app(app: &mut App) -> &mut App { - let filename = - std::env::var("CI_TESTING_CONFIG").unwrap_or_else(|_| "ci_testing_config.ron".to_string()); - let config: CiTestingConfig = ron::from_str( - &std::fs::read_to_string(filename).expect("error reading CI testing configuration file"), - ) - .expect("error deserializing CI testing configuration file"); + #[cfg(not(target_arch = "wasm32"))] + let config: CiTestingConfig = { + let filename = std::env::var("CI_TESTING_CONFIG") + .unwrap_or_else(|_| "ci_testing_config.ron".to_string()); + ron::from_str( + &std::fs::read_to_string(filename) + .expect("error reading CI testing configuration file"), + ) + .expect("error deserializing CI testing configuration file") + }; + #[cfg(target_arch = "wasm32")] + let config: CiTestingConfig = { + let config = include_str!("../../../ci_testing_config.ron"); + ron::from_str(config).expect("error deserializing CI testing configuration file") + }; + app.insert_resource(config) .add_system(ci_testing_exit_after); From 31536044122fb105198ed894a2b44127702bb4aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sat, 21 May 2022 04:42:32 +0200 Subject: [PATCH 2/3] force install wasm-bindgen --- .github/workflows/validation-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validation-jobs.yml b/.github/workflows/validation-jobs.yml index b61f2a37ac687..1996c25a1e9a5 100644 --- a/.github/workflows/validation-jobs.yml +++ b/.github/workflows/validation-jobs.yml @@ -37,7 +37,7 @@ jobs: sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers - name: Install wasm-bindgen - run: cargo install wasm-bindgen-cli + run: cargo install --force wasm-bindgen-cli - name: Run examples shell: bash From 1f8e2fd94119f7e4849476e3155c68163012d3c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sat, 21 May 2022 04:49:51 +0200 Subject: [PATCH 3/3] only run for bors --- .github/bors.toml | 1 + .github/workflows/validation-jobs.yml | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/bors.toml b/.github/bors.toml index 9931de1ec54c6..ddc89f2cd7ddf 100644 --- a/.github/bors.toml +++ b/.github/bors.toml @@ -9,6 +9,7 @@ status = [ "markdownlint", "check-markdown-links", "run-examples", + "run-examples-on-wasm", "check-doc", "check-missing-examples-in-docs", "check-unused-dependencies", diff --git a/.github/workflows/validation-jobs.yml b/.github/workflows/validation-jobs.yml index 1996c25a1e9a5..cc8990773abe1 100644 --- a/.github/workflows/validation-jobs.yml +++ b/.github/workflows/validation-jobs.yml @@ -1,7 +1,11 @@ name: validation jobs on: - push + push: + branches: + - staging + - trying + - main env: CARGO_TERM_COLOR: always