diff --git a/.github/workflows/perfs.yml b/.github/workflows/perfs.yml index 326f916813..e91da8d7c3 100644 --- a/.github/workflows/perfs.yml +++ b/.github/workflows/perfs.yml @@ -1,14 +1,23 @@ name: Performance tests + on: pull_request: - types: [labeled] + types: [opened, synchronize, reopened] + +# Abort if new commit since then +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: perf-tests: - if: ${{ github.event.label.name == 'Performance checks' }} + if: + ${{ !contains(github.event.pull_request.labels.*.name, 'skip-performance-checks') }} runs-on: [ubuntu-latest] + permissions: + pull-requests: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v2 with: @@ -21,4 +30,27 @@ jobs: - run: npm ci - run: export DISPLAY=:99 - run: sudo Xvfb -ac :99 -screen 0 1280x1024x24 > /dev/null 2>&1 & # optional - - run: node tests/performance/run.mjs + - run: + node tests/performance/run.mjs --branch $GITHUB_BASE_REF --remote-git-url + https://github.com/canalplus/rx-player.git --report perf-report.md + - name: Post comment + if: always() + uses: actions/github-script@v7 + with: + script: | + const { readFileSync, existsSync } = require('fs'); + if (!existsSync("./perf-report.md")) { + return; + } + const fileContent = readFileSync("./perf-report.md").toString(); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + // TODO: generate comment header inside the report file instead of here for better portability? + // We should already have access to the sha1 through `git` and the destination branch through the command line arguments. + body: "Automated performance checks have been performed on commit " + + "\`${{github.event.pull_request.head.sha}}\` with the base branch \`${{github.base_ref}}\`.\n\n" + + fileContent + + "\n\n If you want to skip performance checks for latter commits, add the `skip-performance-checks` label to this Pull Request.", + }) diff --git a/.gitignore b/.gitignore index fb71aaae22..4bc3b90f08 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,8 @@ /demo/worker.js /tests/performance/node_modules -/tests/performance/bundle1.js -/tests/performance/bundle2.js +/tests/performance/previous.js +/tests/performance/current.js /tests/performance/package.json /tests/performance/package-lock.json diff --git a/tests/performance/index1.html b/tests/performance/current.html similarity index 80% rename from tests/performance/index1.html rename to tests/performance/current.html index f0e0916f01..fa3eed8cca 100644 --- a/tests/performance/index1.html +++ b/tests/performance/current.html @@ -3,7 +3,7 @@ - + RxPlayer - Performance tests diff --git a/tests/performance/index2.html b/tests/performance/previous.html similarity index 75% rename from tests/performance/index2.html rename to tests/performance/previous.html index 0938519466..8c930565c3 100644 --- a/tests/performance/index2.html +++ b/tests/performance/previous.html @@ -3,7 +3,7 @@ - + RxPlayer - Performance tests diff --git a/tests/performance/run.mjs b/tests/performance/run.mjs index 830a300c88..3c68f189f0 100644 --- a/tests/performance/run.mjs +++ b/tests/performance/run.mjs @@ -6,11 +6,11 @@ import esbuild from "esbuild"; import * as fs from "fs/promises"; import { createServer } from "http"; import * as path from "path"; -import { fileURLToPath } from "url"; +import { fileURLToPath, pathToFileURL } from "url"; import launchStaticServer from "../../scripts/launch_static_server.mjs"; -import getHumanReadableHours from "../../scripts/utils/get_human_readable_hours.mjs"; import removeDir from "../../scripts/utils/remove_dir.mjs"; import createContentServer from "../contents/server.mjs"; +import { appendFileSync, rmSync, writeFileSync } from "fs"; const currentDirectory = path.dirname(fileURLToPath(import.meta.url)); @@ -20,12 +20,17 @@ const CONTENT_SERVER_PORT = 3000; /** Port of the HTTP server which will serve the performance test files */ const PERF_TESTS_PORT = 8080; +/** Port of the HTTP server which will be used to exchange about test results. */ +const RESULT_SERVER_PORT = 6789; + /** * Number of times test are runs on each browser/RxPlayer configuration. * More iterations means (much) more time to perform tests, but also produce * better estimates. + * + * TODO: GitHub actions fails when running the 128th browser. Find out why. */ -const TEST_ITERATIONS = 10; +const TEST_ITERATIONS = 30; /** * After initialization is done, contains the path allowing to run the Chrome @@ -34,12 +39,12 @@ const TEST_ITERATIONS = 10; */ let CHROME_CMD; -/** - * After initialization is done, contains the path allowing to run the Firefox - * browser. - * @type {string|undefined|null} - */ -let FIREFOX_CMD; +// /** +// * After initialization is done, contains the path allowing to run the Firefox +// * browser. +// * @type {string|undefined|null} +// */ +// let FIREFOX_CMD; /** Options used when starting the Chrome browser. */ const CHROME_OPTIONS = [ @@ -56,19 +61,15 @@ const CHROME_OPTIONS = [ "--headless", "--disable-gpu", "--disable-dev-shm-usage", - - // We don't even care about that one but Chrome may not launch without this - // for some unknown reason - "--remote-debugging-port=9222", + "--disk-cache-dir=/dev/null", ]; -/** Options used when starting the Firefox browser. */ -const FIREFOX_OPTIONS = [ - "-no-remote", - "-wait-for-browser", - "-headless", - // "--start-debugger-server 6000", -]; +// /** Options used when starting the Firefox browser. */ +// const FIREFOX_OPTIONS = [ +// "-no-remote", +// "-wait-for-browser", +// "-headless", +// ]; /** * `ChildProcess` instance of the current browser being run. @@ -84,110 +85,393 @@ let currentBrowser; */ const tasks = []; -/** - * Index of the currently ran task in the `tasks` array. - * @see tasks - */ -let nextTaskIndex = 0; - /** * Store results of the performance tests in two arrays: - * - the first one contains the test results of the current RxPlayer version - * - the second one contains the test results of the last RxPlayer version + * - "current" contains the test results of the current RxPlayer version + * - "previous" contains the test results of the last RxPlayer version */ -const allSamples = [[], []]; +const allSamples = { + current: [], + previous: [], +}; + +// If true, this script is called directly +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + const args = process.argv.slice(2); + if (args.includes("-h") || args.includes("--help")) { + displayHelp(); + process.exit(0); + } -/** - * Current results for the tests being run in `currentBrowser`. - * Will be added to `allSamples` once those tests are finished. - */ -let currentTestSample = []; + let branchName; + { + let branchNameIndex = args.indexOf("-b"); + if (branchNameIndex < 0) { + branchNameIndex = args.indexOf("--branch"); + } + if (branchNameIndex >= 0) { + branchName = args[branchNameIndex + 1]; + if (branchName === undefined) { + // eslint-disable-next-line no-console + console.error("ERROR: no branch name provided\n"); + displayHelp(); + process.exit(1); + } + } + } + + let remote; + { + let branchNameIndex = args.indexOf("-u"); + if (branchNameIndex < 0) { + branchNameIndex = args.indexOf("--remote-git-url"); + } + if (branchNameIndex >= 0) { + remote = args[branchNameIndex + 1]; + if (remote === undefined) { + // eslint-disable-next-line no-console + console.error("ERROR: no remote URL provided\n"); + displayHelp(); + process.exit(1); + } + } + } + + let reportFile; + { + let reportFileIndex = args.indexOf("-r"); + if (reportFileIndex < 0) { + reportFileIndex = args.indexOf("--report"); + } + if (reportFileIndex >= 0) { + reportFile = args[reportFileIndex + 1]; + if (reportFile === undefined) { + // eslint-disable-next-line no-console + console.error("ERROR: no file path provided\n"); + displayHelp(); + process.exit(1); + } + } + } + + /* eslint-disable no-console */ + if (reportFile !== undefined) { + try { + console.log(`Removing previous report file if it exists ("${reportFile}")`); + rmSync(reportFile); + } catch (_) { + // We don't really care here + } + } + + initializePerformanceTestsPages({ + branchName: branchName ?? "dev", + remoteGitUrl: remote, + }) + .then(() => runPerformanceTests()) + .then(async (results) => { + if (reportFile !== undefined) { + try { + writeFileSync(reportFile, "Tests results\n" + "-------------\n\n"); + if (results.worse.length === 0) { + appendToReportFile("✅ Tests have passed.\n"); + } else { + appendToReportFile("❌ Tests have failed.\n"); + } + appendToReportFile( + "Performance tests 1st run output\n" + "--------------------------------", + ); + } catch (err) { + console.error( + `Cannot write file output: Invalid file path given: ${reportFile}`, + ); + } + } + + if (results.worse.length > 0) { + const failureTxt = + "\nWorse performance for tests:\n\n" + + formatResultInHumanReadableWay(results.worse); + console.warn(failureTxt); + appendToReportFile(failureTxt); + } + + if (results.better.length > 0) { + const betterTxt = + "\nBetter performance for tests:\n\n" + + formatResultInHumanReadableWay(results.better); + console.log(betterTxt); + appendToReportFile(betterTxt); + } + + if (results.notSignificative.length > 0) { + const notSignificativeTxt = + "\nNo significative change in performance for tests:\n\n" + + formatResultInHumanReadableWay(results.notSignificative); + console.log(notSignificativeTxt); + appendToReportFile(notSignificativeTxt); + } + + if (results.worse.length === 0) { + process.exit(0); + } + + console.warn("\nRetrying one time just to check if unlucky..."); + + const results2 = await runPerformanceTests(); + console.error("\nFinal result after 2 attempts\n-----------------------------\n"); + appendToReportFile( + "\nPerformance tests 2nd run output\n" + "--------------------------------", + ); + + if (results.better.length > 0) { + console.error( + "\nBetter performance at first attempt for tests:\n\n" + + formatResultInHumanReadableWay(results.better), + ); + } + if (results2.better.length > 0) { + const betterTxt = + "\nBetter performance for tests:\n\n" + + formatResultInHumanReadableWay(results.better); + appendToReportFile(betterTxt); + console.error( + "\nBetter performance at second attempt for tests:\n\n" + + formatResultInHumanReadableWay(results2.better), + ); + } + + if (results.worse.length > 0) { + console.error( + "\nWorse performance at first attempt for tests:\n\n" + + formatResultInHumanReadableWay(results.worse), + ); + } + if (results2.worse.length > 0) { + const failureTxt = + "\nWorse performance at second attempt for tests:\n\n" + + formatResultInHumanReadableWay(results.worse); + console.warn(failureTxt); + appendToReportFile(failureTxt); + } + + if (results2.notSignificative.length > 0) { + const notSignificativeTxt = + "\nNo significative change in performance for tests:\n\n" + + formatResultInHumanReadableWay(results.notSignificative); + appendToReportFile(notSignificativeTxt); + } + + for (const failure1 of results.worse) { + if (results2.worse.some((r) => r.testName === failure1.testName)) { + process.exit(1); + } + } + process.exit(0); + + function appendToReportFile(text) { + if (reportFile === undefined) { + return; + } + try { + appendFileSync(reportFile, text + "\n"); + } catch (err) { + /* eslint-disable-next-line no-console */ + console.error( + `Cannot write file output: Invalid file path given: ${reportFile}`, + ); + } + } + }) + .catch((err) => { + console.error("Error:", err); + return process.exit(1); + }); + /* eslint-enable no-console */ +} /** - * Contains references to every launched servers, with a `close` method allowing - * to close each one of them. + * Take test results as outputed by performance tests and output a markdown + * table listing them in hopefully a readable way. + * @param {Array.} results + * @returns {string} */ -const servers = []; +function formatResultInHumanReadableWay(results) { + if (results.length === 0) { + return ""; + } + const testNames = results.map((r) => r.testName); + const meanResult = results.map( + (r) => + `${r.previousMean.toFixed(2)}ms -> ${r.currentMean.toFixed(2)}ms ` + + `(${r.meanDifferenceMs.toFixed(3)}ms, z: ${r.zScore.toFixed(5)})`, + ); + const medianResult = results.map( + (r) => `${r.previousMedian.toFixed(2)}ms -> ${r.currentMedian.toFixed(2)}ms`, + ); + + const nameColumnInnerLength = Math.max( + testNames.reduce((acc, t) => Math.max(acc, t.length), 0) + 2 /* margin */, + " Name ".length, + ); + const meanColumnInnerLength = Math.max( + meanResult.reduce((acc, t) => Math.max(acc, t.length), 0) + 2 /* margin */, + " Mean ".length, + ); + const medianColumnInnerLength = Math.max( + medianResult.reduce((acc, t) => Math.max(acc, t.length), 0) + 2 /* margin */, + " Median ".length, + ); + + let str; + + { + // Table header + const nameWhitespaceLength = (nameColumnInnerLength - "Name".length) / 2; + const meanWhitespaceLength = (meanColumnInnerLength - "Mean".length) / 2; + const medianWhitespaceLength = (medianColumnInnerLength - "Median".length) / 2; + str = + "|" + + " ".repeat(Math.floor(nameWhitespaceLength)) + + "Name" + + " ".repeat(Math.ceil(nameWhitespaceLength)) + + "|" + + " ".repeat(Math.floor(meanWhitespaceLength)) + + "Mean" + + " ".repeat(Math.ceil(meanWhitespaceLength)) + + "|" + + " ".repeat(Math.floor(medianWhitespaceLength)) + + "Median" + + " ".repeat(Math.ceil(medianWhitespaceLength)) + + "|\n" + + "|" + + "-".repeat(nameColumnInnerLength) + + "|" + + "-".repeat(meanColumnInnerLength) + + "|" + + "-".repeat(medianColumnInnerLength) + + "|"; + } + for (let i = 0; i < results.length; i++) { + str += "\n"; + const nameWhitespaceLength = (nameColumnInnerLength - testNames[i].length) / 2; + const meanWhitespaceLength = (meanColumnInnerLength - meanResult[i].length) / 2; + const medianWhitespaceLength = (medianColumnInnerLength - medianResult[i].length) / 2; + str += + "|" + + " ".repeat(Math.floor(nameWhitespaceLength)) + + testNames[i] + + " ".repeat(Math.ceil(nameWhitespaceLength)) + + "|" + + " ".repeat(Math.floor(meanWhitespaceLength)) + + meanResult[i] + + " ".repeat(Math.ceil(meanWhitespaceLength)) + + "|" + + " ".repeat(Math.floor(medianWhitespaceLength)) + + medianResult[i] + + " ".repeat(Math.ceil(medianWhitespaceLength)) + + "|"; + } + return str; +} /** - * Callback called when all current tasks are finished. - * This allows to perform several groups of tasks (e.g. per browser). + * Initialize and start all tests on Chrome. + * @returns {Promise.} */ -let onFinished = () => {}; +function runPerformanceTests() { + return new Promise((resolve, reject) => { + let isFinished = false; + let contentServer; + let resultServer; + let staticServer; + + const onFinished = () => { + isFinished = true; + closeServers(); + const results = compareSamples(); + closeBrowser().catch((err) => { + // eslint-disable-next-line no-console + console.error("Failed to close the browser:", err); + }); + resolve(results); + }; + const onError = (error) => { + isFinished = true; + closeServers(); + closeBrowser().catch((err) => { + // eslint-disable-next-line no-console + console.error("Failed to close the browser:", err); + }); + reject(error); + }; -start().catch((err) => { - // eslint-disable-next-line no-console - console.error("Error:", err); - return process.exit(1); -}); - -/** Initialize and start all tests on Chrome. */ -async function start() { - await initScripts(); - await initServers(); - - onFinished = () => { - const hasSucceededOnChrome = compareSamples(); - shutdown().catch((err) => { - // eslint-disable-next-line no-console - console.error("Failed to shutdown:", err); - }); - if (!hasSucceededOnChrome) { - // eslint-disable-next-line no-console - console.error("Tests failed on Chrome"); - return process.exit(1); - } - return process.exit(0); - - // TODO also run on Firefox? Despite my efforts, I did not succeed to run - // tests on it. - // onFinished = async () => { - // shutdown(); - // const hasSucceededOnFirefox = compareSamples(); - // if (!hasSucceededOnChrome || !hasSucceededOnFirefox) { - // // eslint-disable-next-line no-console - // console.error("Tests failed on:" + - // (!hasSucceededOnChrome ? " Chrome" : "") + - // (!hasSucceededOnFirefox ? " Firefox" : "")); - // return process.exit(1); - // } - // return process.exit(0); - // }; - // startAllTestsOnFirefox(); - }; + const closeServers = () => { + contentServer?.close(); + contentServer = undefined; + resultServer?.close(); + resultServer = undefined; + staticServer?.close(); + staticServer = undefined; + }; - startAllTestsOnChrome().catch((err) => { - // eslint-disable-next-line no-console - console.error("Error:", err); - return process.exit(1); + initServers(onFinished, onError) + .then((servers) => { + contentServer = servers.contentServer; + resultServer = servers.resultServer; + staticServer = servers.staticServer; + if (isFinished) { + closeServers(); + } + return startAllTestsOnChrome(); + }) + .catch(onError); }); } /** * Initialize all servers used for the performance tests. + * @param {Function} onFinished + * @param {function} onError * @returns {Promise} - Resolves when all servers are listening. */ -async function initServers() { - const contentServer = createContentServer(CONTENT_SERVER_PORT); - const staticServer = launchStaticServer(currentDirectory, { - httpPort: PERF_TESTS_PORT, - }); - const resultServer = createResultServer(); - servers.push(contentServer, staticServer, resultServer); - await Promise.all([ - contentServer.listeningPromise, - staticServer.listeningPromise, - resultServer.listeningPromise, - ]); +async function initServers(onFinished, onError) { + let contentServer; + let staticServer; + let resultServer; + try { + contentServer = createContentServer(CONTENT_SERVER_PORT); + staticServer = launchStaticServer(currentDirectory, { + httpPort: PERF_TESTS_PORT, + }); + resultServer = createResultServer(onFinished, onError); + await Promise.all([ + contentServer.listeningPromise, + staticServer.listeningPromise, + resultServer.listeningPromise, + ]); + return { contentServer, resultServer, staticServer }; + } catch (error) { + contentServer?.close(); + staticServer?.close(); + resultServer?.close(); + throw error; + } } /** * Prepare all scripts needed for the performance tests. + * @param {Object} opts - Various options for scripts initialization. + * @param {string} opts.branchName - The name of the branch results should be + * compared to. + * @param {string} [opts.remoteGitUrl] - The git URL where the current + * repository can be cloned for comparisons. + * The one for the current git repository by default. * @returns {Promise} - Resolves when the initialization is finished. */ -async function initScripts() { +async function initializePerformanceTestsPages({ branchName, remoteGitUrl }) { + await prepareLastRxPlayerTests({ branchName, remoteGitUrl }); await prepareCurrentRxPlayerTests(); - await prepareLastRxPlayerTests(); } /** @@ -196,16 +480,22 @@ async function initScripts() { */ async function prepareCurrentRxPlayerTests() { await linkCurrentRxPlayer(); - await createBundle({ output: "bundle1.js", minify: false, production: true }); + await createBundle({ output: "current.js", minify: false, production: true }); } /** * Build test file for testing the last version of the RxPlayer. + * @param {Object} opts - Various options. + * @param {string} opts.branchName - The name of the branch results should be + * compared to. + * @param {string} [opts.remoteGitUrl] - The git URL where the current + * repository can be cloned for comparisons. + * The one for the current git repository by default. * @returns {Promise} */ -async function prepareLastRxPlayerTests() { - await linkLastRxPlayer(); - await createBundle({ output: "bundle2.js", minify: false, production: true }); +async function prepareLastRxPlayerTests({ branchName, remoteGitUrl }) { + await linkRxPlayerBranch({ branchName, remoteGitUrl }); + await createBundle({ output: "previous.js", minify: false, production: true }); } /** @@ -214,188 +504,146 @@ async function prepareLastRxPlayerTests() { * @returns {Promise} */ async function linkCurrentRxPlayer() { - await removeDir(path.join(currentDirectory, "node_modules")); - await fs.mkdir(path.join(currentDirectory, "node_modules")); + const rootDir = path.join(currentDirectory, "..", ".."); + await removeDir(path.join(rootDir, "dist")); + + const innerNodeModulesPath = path.join(currentDirectory, "node_modules"); + await removeDir(innerNodeModulesPath); + await fs.mkdir(innerNodeModulesPath); + const rxPlayerPath = path.join(innerNodeModulesPath, "rx-player"); await spawnProc( "npm run build", [], - (code) => new Error(`npm install exited with code ${code}`), + (code) => new Error(`npm run build exited with code ${code}`), ).promise; - await fs.symlink( - path.join(currentDirectory, "..", ".."), - path.join(currentDirectory, "node_modules", "rx-player"), - ); + await fs.symlink(path.join(currentDirectory, "..", ".."), rxPlayerPath); } /** * Link the last published RxPlayer version to the performance tests, so * performance of new code can be compared to it. + * @param {Object} opts - Various options. + * @param {string} opts.branchName - The name of the branch results should be + * compared to. + * @param {string} [opts.remoteGitUrl] - The git URL where the current + * repository can be cloned for comparisons. + * The one for the current git repository by default. * @returns {Promise} */ -async function linkLastRxPlayer() { - await removeDir(path.join(currentDirectory, "node_modules")); +async function linkRxPlayerBranch({ branchName, remoteGitUrl }) { + const rootDir = path.join(currentDirectory, "..", ".."); + await removeDir(path.join(rootDir, "dist")); + + const innerNodeModulesPath = path.join(currentDirectory, "node_modules"); + await removeDir(innerNodeModulesPath); + await fs.mkdir(innerNodeModulesPath); + const rxPlayerPath = path.join(innerNodeModulesPath, "rx-player"); + let url = + remoteGitUrl ?? + (await execCommandAndGetFirstOutput("git config --get remote.origin.url")); + url = url.trim(); + await spawnProc( + `git clone -b ${branchName} ${url} ${rxPlayerPath}`, + [], + (code) => new Error(`git clone exited with code ${code}`), + ).promise; + await spawnProc( + `cd ${rxPlayerPath} && npm install`, + [], + (code) => new Error(`npm install failed with code ${code}`), + ).promise; await spawnProc( - "npm install", - ["--prefix", currentDirectory, "rx-player"], - (code) => new Error(`npm install exited with code ${code}`), + `cd ${rxPlayerPath} && npm run build`, + [], + (code) => new Error(`npm run build exited with code ${code}`), ).promise; + + // GitHub actions, for unknown reasons, want to use the root's `dist` directory + // TODO: find why + await fs.symlink( + path.join(rxPlayerPath, "dist"), + path.join(currentDirectory, "..", "..", "dist"), + ); } /** * Build the `tasks` array and start all tests on the Chrome browser. - * The `onFinished` callback will be called when finished. + * @returns {Promise} */ async function startAllTestsOnChrome() { CHROME_CMD = await getChromeCmd(); + tasks.length = 0; for (let i = 0; i < TEST_ITERATIONS; i++) { - tasks.push(() => startCurrentPlayerTestsOnChrome(i * 2, TEST_ITERATIONS * 2)); - tasks.push(() => startLastPlayerTestsOnChrome(i * 2 + 1, TEST_ITERATIONS * 2)); + tasks.push(() => startTestsOnChrome(i % 2 === 0, i + 1, TEST_ITERATIONS)); } if (CHROME_CMD === null) { - // eslint-disable-next-line no-console - console.error("Error: Chrome not found on the current platform"); - return process.exit(1); - } - startNextTaskOrFinish().catch((err) => { - // eslint-disable-next-line no-console - console.error("Error:", err); - return process.exit(1); - }); -} - -/** - * Build the `tasks` array and start all tests on the Chrome browser. - * The `onFinished` callback will be called when finished. - * TODO Find out why Firefox just fails without running tests. - */ -// eslint-disable-next-line no-unused-vars -async function _startAllTestsOnFirefox() { - FIREFOX_CMD = await getFirefoxCmd(); - for (let i = 0; i < TEST_ITERATIONS; i++) { - tasks.push(() => startCurrentPlayerTestsOnFirefox(i * 2, TEST_ITERATIONS * 2)); - tasks.push(() => startLastPlayerTestsOnFirefox(i * 2 + 1, TEST_ITERATIONS * 2)); + throw new Error("Error: Chrome not found on the current platform"); } - if (FIREFOX_CMD === null) { - // eslint-disable-next-line no-console - console.error("Error: Firefox not found on the current platform"); - return process.exit(1); + if (tasks.length === 0) { + throw new Error("No task scheduled"); } - startNextTaskOrFinish(); + return tasks.shift()(); } /** * Free all resources and terminate script. */ -async function shutdown() { +async function closeBrowser() { if (currentBrowser !== undefined) { - currentBrowser.kill(); + currentBrowser.kill("SIGKILL"); currentBrowser = undefined; } - while (servers.length > 0) { - servers.pop().close(); - } } /** * Starts the next function in the `tasks` array. * If no task are available anymore, call the `onFinished` callback. + * @param {Function} onFinished */ -function startNextTaskOrFinish() { - if (nextTaskIndex > 0) { - allSamples[(nextTaskIndex - 1) % 2].push(...currentTestSample); - } - currentTestSample = []; - if (tasks[nextTaskIndex] === undefined) { +function startNextTaskOrFinish(onFinished) { + const nextTask = tasks.shift(); + if (nextTask === undefined) { onFinished(); } - nextTaskIndex++; - return tasks[nextTaskIndex - 1](); -} - -/** - * Start Chrome browser running performance tests on the current RxPlayer - * version. - * @returns {Promise} - */ -async function startCurrentPlayerTestsOnChrome(testNb, testTotal) { - // eslint-disable-next-line no-console - console.log( - "Running tests on Chrome on the current RxPlayer version " + - `(${testNb}/${testTotal})`, - ); - startPerfhomepageOnChrome("index1.html").catch((err) => { - // eslint-disable-next-line no-console - console.error("Could not launch page on Chrome:", err); - process.exit(1); - }); -} - -/** - * Start Firefox browser running performance tests on the current RxPlayer - * version. - * @returns {Promise} - */ -async function startCurrentPlayerTestsOnFirefox(testNb, testTotal) { - // eslint-disable-next-line no-console - console.log( - "Running tests on Firefox on the current RxPlayer version " + - `(${testNb}/${testTotal})`, - ); - startPerfhomepageOnFirefox("index1.html").catch((err) => { - // eslint-disable-next-line no-console - console.error("Could not launch page on Firefox:", err); - process.exit(1); - }); -} - -/** - * Start Chrome browser running performance tests on the last published RxPlayer - * version. - * @returns {Promise} - */ -async function startLastPlayerTestsOnChrome(testNb, testTotal) { - // eslint-disable-next-line no-console - console.log( - "Running tests on Chrome on the previous RxPlayer version " + - `(${testNb}/${testTotal})`, - ); - startPerfhomepageOnChrome("index2.html").catch((err) => { - // eslint-disable-next-line no-console - console.error("Could not launch page on Chrome:", err); - process.exit(1); - }); + return nextTask(); } /** - * Start Firefox browser running performance tests on the last published - * RxPlayer version. + * Start Chrome browser running performance tests. + * @param {boolean} startWithCurrent - If `true` we will begin with tests on the + * current build. If `false` we will start with the previous build. We will + * then alternate. + * The global idea is to ensure we're testing both cases as to remove some + * potential for lower performances due e.g. to browser internal logic. + * @param {number} testNb - The current test iteration, starting from `1` to + * `testTotal`. Used to indicate progress. + * @param {number} testTotal - The maximum number of iterations. Used to + * indicate progress. * @returns {Promise} */ -async function startLastPlayerTestsOnFirefox(testNb, testTotal) { +async function startTestsOnChrome(startWithCurrent, testNb, testTotal) { // eslint-disable-next-line no-console - console.log( - "Running tests on Firefox on the previous RxPlayer version " + - `(${testNb}/${testTotal})`, - ); - startPerfhomepageOnFirefox("index2.html").catch((err) => { - // eslint-disable-next-line no-console - console.error("Could not launch page on Firefox:", err); - process.exit(1); + console.log(`Running tests on Chrome (${testNb}/${testTotal})`); + return startPerfhomepageOnChrome( + startWithCurrent + ? `current.html#p=${RESULT_SERVER_PORT};` + : `previous.html#p=${RESULT_SERVER_PORT};`, + ).catch((err) => { + throw new Error("Could not launch page on Chrome: " + err.toString()); }); } /** * Start the performance tests on Chrome. * Set `currentBrowser` to chrome. + * @param {string} homePage - Page on which to run the browser. */ async function startPerfhomepageOnChrome(homePage) { if (currentBrowser !== undefined) { - currentBrowser.kill(); + currentBrowser.kill("SIGKILL"); } if (CHROME_CMD === undefined || CHROME_CMD === null) { - // eslint-disable-next-line no-console - console.error("Error: Starting browser before initialization"); - return process.exit(1); + throw new Error("Starting browser before initialization"); } const spawned = spawnProc(CHROME_CMD, [ ...CHROME_OPTIONS, @@ -404,35 +652,17 @@ async function startPerfhomepageOnChrome(homePage) { currentBrowser = spawned.child; } -/** - * Start the performance tests on Firefox. - * Set `currentBrowser` to Firefox. - */ -async function startPerfhomepageOnFirefox(homePage) { - if (currentBrowser !== undefined) { - currentBrowser.kill(); - } - if (FIREFOX_CMD === undefined || FIREFOX_CMD === null) { - // eslint-disable-next-line no-console - console.error("Error: Starting browser before initialization"); - return process.exit(1); - } - const spawned = spawnProc(FIREFOX_CMD, [ - ...FIREFOX_OPTIONS, - `http://localhost:${PERF_TESTS_PORT}/${homePage}`, - ]); - currentBrowser = spawned.child; -} - /** * Create HTTP server which will receive test results and react appropriately. + * @param {Function} onFinished + * @param {function} onError * @returns {Object} */ -function createResultServer() { +function createResultServer(onFinished, onError) { const server = createServer(onRequest); return { listeningPromise: new Promise((res) => { - server.listen(6789, function () { + server.listen(RESULT_SERVER_PORT, function () { res(); }); }), @@ -455,16 +685,30 @@ function createResultServer() { const parsedBody = JSON.parse(body); if (parsedBody.type === "log") { // eslint-disable-next-line no-console - console.log("LOG:", parsedBody.data); + console.warn("LOG:", parsedBody.data); + } else if (parsedBody.type === "error") { + onError(new Error("ERROR: A fatal error happened: " + parsedBody.data)); + return; } else if (parsedBody.type === "done") { if (currentBrowser !== undefined) { - currentBrowser.kill(); + currentBrowser.kill("SIGKILL"); currentBrowser = undefined; } - displayTemporaryResults(); - startNextTaskOrFinish(); - } else { - currentTestSample.push(parsedBody.data); + if (allSamples.previous.length > 0 && allSamples.current.length > 0) { + compareSamples(); + } + startNextTaskOrFinish(onFinished).catch(onError); + } else if (parsedBody.type === "value") { + let page; + if (parsedBody.page === "current") { + page = "current"; + } else if (parsedBody.page === "previous") { + page = "previous"; + } else { + onError(new Error("Unknown page: " + parsedBody.page)); + return; + } + allSamples[page].push(parsedBody.data); } answerWithCORS(response, 200, "OK"); return; @@ -545,73 +789,112 @@ function rankSamples(list) { * Compare both elements of `allSamples` and display comparative results. * Returns false if any of the tested scenario had a significant performance * regression. - * @returns {boolean} + * @returns {Object} */ function compareSamples() { - if (allSamples.length !== 2) { - throw new Error("Not enough result"); - } - const samplesPerScenario = [ - getSamplePerScenarios(allSamples[0]), - getSamplePerScenarios(allSamples[1]), - ]; - - let hasSucceeded = true; - for (const testName of Object.keys(samplesPerScenario[0])) { - const sample1 = samplesPerScenario[0][testName]; - const sample2 = samplesPerScenario[1][testName]; - if (sample2 === undefined) { + const samplesPerScenario = { + current: getSamplePerScenarios(allSamples.current), + previous: getSamplePerScenarios(allSamples.previous), + }; + + const results = { + worse: [], + better: [], + notSignificative: [], + }; + for (const testName of Object.keys(samplesPerScenario.current)) { + const sampleCurrent = samplesPerScenario.current[testName]; + const samplePrevious = samplesPerScenario.previous[testName]; + if (samplePrevious === undefined) { // eslint-disable-next-line no-console console.error("Error: second result misses a scenario:", testName); continue; } - const result1 = getResultsForSample(sample1); - const result2 = getResultsForSample(sample2); - - // eslint-disable-next-line no-console - console.log("For current Player:\n" + "==================="); - // eslint-disable-next-line no-console - console.log( - `test name: ${testName}\n` + - `mean: ${result1.mean}\n` + - `variance: ${result1.variance}\n` + - `standardDeviation: ${result1.standardDeviation}\n` + - `standardErrorOfMean: ${result1.standardErrorOfMean}\n` + - `moe: ${result1.moe}\n`, + const resultCurrent = getResultsForSample(sampleCurrent); + const resultPrevious = getResultsForSample(samplePrevious); + + const medianDiffMs = resultPrevious.median - resultCurrent.median; + const meanDiffMs = resultPrevious.mean - resultCurrent.mean; + const uValue = getUValueFromSamples(sampleCurrent, samplePrevious); + const zScore = Math.abs( + calculateZScore(uValue, sampleCurrent.length, samplePrevious.length), ); - - // eslint-disable-next-line no-console - console.log("\nFor previous Player:\n" + "==================="); - // eslint-disable-next-line no-console - console.log( - `test name: ${testName}\n` + - `mean: ${result2.mean}\n` + - `variance: ${result2.variance}\n` + - `standardDeviation: ${result2.standardDeviation}\n` + - `standardErrorOfMean: ${result2.standardErrorOfMean}\n` + - `moe: ${result2.moe}\n`, - ); - - const difference = (result2.mean - result1.mean) / result1.mean; - - // eslint-disable-next-line no-console - console.log(`\nDifference: ${difference * 100}`); - - const uValue = getUValueFromSamples(sample1, sample2); - const zScore = Math.abs(calculateZScore(uValue, sample1.length, sample2.length)); - const isSignificant = zScore > 1.96; + // For p-value of 5% + // const isSignificant = zScore > 1.96; + // For p-value of 1% + const isSignificant = zScore > 2.575829; + + /* eslint-disable no-console */ + console.log(""); + console.log(`> Current results for test:`, testName); + console.log(""); + console.log(" For current Player:"); + console.log(` mean: ${resultCurrent.mean}`); + console.log(` median: ${resultCurrent.median}`); + console.log(` variance: ${resultCurrent.variance}`); + console.log(` standard deviation: ${resultCurrent.standardDeviation}`); + console.log(` standard error of mean: ${resultCurrent.standardErrorOfMean}`); + console.log(` moe: ${resultCurrent.moe}`); + console.log(""); + console.log(" For previous Player:"); + console.log(` mean: ${resultPrevious.mean}`); + console.log(` median: ${resultPrevious.median}`); + console.log(` variance: ${resultPrevious.variance}`); + console.log(` standard deviation: ${resultPrevious.standardDeviation}`); + console.log(` standard error of mean: ${resultPrevious.standardErrorOfMean}`); + console.log(` moe: ${resultPrevious.moe}`); + console.log(""); + console.log(" Results"); + console.log(` mean difference time (negative is slower): ${meanDiffMs} ms`); if (isSignificant) { - // eslint-disable-next-line no-console - console.log(`The difference is significant (z: ${zScore})`); - if (difference < 0) { - hasSucceeded = false; + console.log(` The difference is significant (z: ${zScore})`); + if (meanDiffMs < -2 && medianDiffMs < -2) { + results.worse.push({ + testName, + previousMean: resultPrevious.mean, + currentMean: resultCurrent.mean, + previousMedian: resultPrevious.median, + currentMedian: resultCurrent.median, + meanDifferenceMs: meanDiffMs, + zScore, + }); + } else if (meanDiffMs > 2 && medianDiffMs > 2) { + results.better.push({ + testName, + previousMean: resultPrevious.mean, + currentMean: resultCurrent.mean, + previousMedian: resultPrevious.median, + currentMedian: resultCurrent.median, + meanDifferenceMs: meanDiffMs, + zScore, + }); + } else { + results.notSignificative.push({ + testName, + previousMean: resultPrevious.mean, + currentMean: resultCurrent.mean, + previousMedian: resultPrevious.median, + currentMedian: resultCurrent.median, + meanDifferenceMs: meanDiffMs, + zScore, + }); } } else { - // eslint-disable-next-line no-console - console.log(`The difference is not significant (z: ${zScore})`); + console.log(` The difference is not significant (z: ${zScore})`); + results.notSignificative.push({ + testName, + previousMean: resultPrevious.mean, + currentMean: resultCurrent.mean, + previousMedian: resultPrevious.median, + currentMedian: resultCurrent.median, + meanDifferenceMs: meanDiffMs, + zScore, + }); } + console.log(""); } - return hasSucceeded; + /* eslint-enable no-console */ + return results; function calculateZScore(u, len1, len2) { return (u - (len1 * len2) / 2) / Math.sqrt((len1 * len2 * (len1 + len2 + 1)) / 12); } @@ -619,18 +902,18 @@ function compareSamples() { /** * Calculate U value from the Mann–Whitney U test from two samples. - * @param {Array.} sample1 - * @param {Array.} sample2 + * @param {Array.} sampleCurrent + * @param {Array.} samplePrevious * @returns {number} */ -function getUValueFromSamples(sample1, sample2) { - const concatSamples = sample1.concat(sample2); +function getUValueFromSamples(sampleCurrent, samplePrevious) { + const concatSamples = sampleCurrent.concat(samplePrevious); const ranked = rankSamples(concatSamples); - const summedRanks1 = sumRanks(ranked, sample1); - const summedRanks2 = sumRanks(ranked, sample2); - const n1 = sample1.length; - const n2 = sample2.length; + const summedRanks1 = sumRanks(ranked, sampleCurrent); + const summedRanks2 = sumRanks(ranked, samplePrevious); + const n1 = sampleCurrent.length; + const n2 = samplePrevious.length; const u1 = calculateUValue(summedRanks1, n1, n2); const u2 = calculateUValue(summedRanks2, n2, n1); @@ -662,6 +945,16 @@ function getUValueFromSamples(sample1, sample2) { * @returns {Object} */ function getResultsForSample(sample) { + sample.sort(); + let median; + if (sample.length === 0) { + median = 0; + } else { + median = + sample.length % 2 === 0 + ? sample[sample.length / 2 - 1] + sample[sample.length / 2] / 2 + : sample[Math.floor(sample.length / 2)]; + } const mean = sample.reduce((acc, x) => acc + x, 0) / sample.length; const variance = sample.reduce((acc, x) => { @@ -672,7 +965,7 @@ function getResultsForSample(sample) { const standardErrorOfMean = standardDeviation / Math.sqrt(sample.length); const criticalVal = 1.96; const moe = standardErrorOfMean * criticalVal; - return { mean, variance, standardErrorOfMean, standardDeviation, moe }; + return { mean, median, variance, standardErrorOfMean, standardDeviation, moe }; } /** @@ -694,28 +987,6 @@ function getSamplePerScenarios(samplesObj) { }, {}); } -/** - * Log results for `currentTestSample`: mean, standard deviation etc. - */ -function displayTemporaryResults() { - const testedScenarios = getSamplePerScenarios(currentTestSample); - for (const testName of Object.keys(testedScenarios)) { - const scenarioSample = testedScenarios[testName]; - const results = getResultsForSample(scenarioSample); - // eslint-disable-next-line no-console - console.log( - `test name: ${testName}\n` + - `mean: ${results.mean}\n` + - `first sample: ${scenarioSample[0]}\n` + - `last sample: ${scenarioSample[scenarioSample.length - 1]}\n` + - `variance: ${results.variance}\n` + - `standard deviation: ${results.standardDeviation}\n` + - `standard error of mean: ${results.standardErrorOfMean}\n` + - `moe: ${results.moe}\n`, - ); - } -} - /** * Build the performance tests. * @param {Object} options @@ -728,46 +999,32 @@ function displayTemporaryResults() { function createBundle(options) { const minify = !!options.minify; const isDevMode = !options.production; - return new Promise((res) => { - esbuild - .build({ - entryPoints: [path.join(currentDirectory, "src", "main.js")], - bundle: true, - minify, - outfile: path.join(currentDirectory, options.output), - define: { - __TEST_CONTENT_SERVER__: JSON.stringify({ - URL: "127.0.0.1", - PORT: "3000", - }), - "process.env.NODE_ENV": JSON.stringify( - isDevMode ? "development" : "production", - ), - __ENVIRONMENT__: JSON.stringify({ - PRODUCTION: 0, - DEV: 1, - CURRENT_ENV: isDevMode ? 1 : 0, - }), - __LOGGER_LEVEL__: JSON.stringify({ - CURRENT_LEVEL: "INFO", - }), - __GLOBAL_SCOPE__: JSON.stringify(false), - }, - }) - .then( - () => { - res(); - }, - (err) => { - // eslint-disable-next-line no-console - console.error( - `\x1b[31m[${getHumanReadableHours()}]\x1b[0m Demo build failed:`, - err, - ); - process.exit(1); - }, - ); - }); + return esbuild + .build({ + entryPoints: [path.join(currentDirectory, "src", "main.js")], + bundle: true, + minify, + outfile: path.join(currentDirectory, options.output), + define: { + __TEST_CONTENT_SERVER__: JSON.stringify({ + URL: "127.0.0.1", + PORT: "3000", + }), + "process.env.NODE_ENV": JSON.stringify(isDevMode ? "development" : "production"), + __ENVIRONMENT__: JSON.stringify({ + PRODUCTION: 0, + DEV: 1, + CURRENT_ENV: isDevMode ? 1 : 0, + }), + __LOGGER_LEVEL__: JSON.stringify({ + CURRENT_LEVEL: "INFO", + }), + __GLOBAL_SCOPE__: JSON.stringify(false), + }, + }) + .catch((err) => { + throw new Error(`Demo build failed:`, err); + }); } /** @@ -779,15 +1036,13 @@ function createBundle(options) { function spawnProc(command, args, errorOnCode) { let child; const prom = new Promise((res, rej) => { - child = spawn(command, args, { shell: true, stdio: "inherit" }).on( - "close", - (code) => { - if (code !== 0 && typeof errorOnCode === "function") { - rej(errorOnCode(code)); - } - res(); - }, - ); + child = spawn(command, args, { shell: true, stdio: "inherit" }); + child.on("close", (code) => { + if (code !== 0 && typeof errorOnCode === "function") { + rej(errorOnCode(code)); + } + res(); + }); }); return { promise: prom, @@ -841,28 +1096,24 @@ async function getChromeCmd() { return null; } default: - // eslint-disable-next-line no-console - console.error("Error: unsupported platform:", process.platform); - process.exit(1); - } -} - -/** - * Returns string corresponding to the Chrome binary. - * @returns {Promise.} - */ -async function getFirefoxCmd() { - switch (process.platform) { - case "linux": { - return "firefox"; - } - // TODO other platforms - default: - // eslint-disable-next-line no-console - console.error("Error: unsupported platform:", process.platform); - process.exit(1); + throw new Error("Error: unsupported platform:", process.platform); } } +// +// /** +// * Returns string corresponding to the Chrome binary. +// * @returns {Promise.} +// */ +// async function getFirefoxCmd() { +// switch (process.platform) { +// case "linux": { +// return "firefox"; +// } +// // TODO other platforms +// default: +// throw new Error("Error: unsupported platform:", process.platform); +// } +// } function execCommandAndGetFirstOutput(command) { return new Promise((res, rej) => { @@ -875,3 +1126,22 @@ function execCommandAndGetFirstOutput(command) { }); }); } + +/** + * Display through `console.log` an helping message relative to how to run this + * script. + */ +function displayHelp() { + /* eslint-disable-next-line no-console */ + console.log( + `Usage: node run.mjs [options] +Available options: + -h, --help Display this help message + -b , --branch Specify the branch name the performance results should be compared to. + Defaults to the "dev" branch., + -u , --remote-git-url Specify the remote git URL where the current repository can be cloned from. + Defaults to the current remote URL. + -r , --report Optional path to markdown file where a report will be written in a + human-readable way once done.`, + ); +} diff --git a/tests/performance/src/lib.js b/tests/performance/src/lib.js new file mode 100644 index 0000000000..6ad54c4933 --- /dev/null +++ b/tests/performance/src/lib.js @@ -0,0 +1,250 @@ +/** + * "Tests" in performance tests are simple time measurements between a "start" + * and an "end" event. `pendingTests` is a map whose keys are the name of the + * tests that have been started but not yet ended and whose the values are the + * result of `performance.now()` when they where started. + */ +const pendingTests = new Map(); + +/** + * Tests are grouped into a collection which make sense together and which may + * be associated to a timeout value. + * This array list objects, each describing a particular group. See usage. + */ +const groups = []; + +/** + * When `true`, we began tests for the current page, and are thus not able to + * register new test groups anymore. + */ +let areTestsAlreadyRunning = false; + +const hashComponents = parseUrlHash(); +const resultServerPort = parseInt(hashComponents.p); +const tryAttempt = hashComponents.t === undefined ? 1 : parseInt(hashComponents.t); + +if (isNaN(resultServerPort)) { + throw new Error("The current page should have a valid result server port in its URL"); +} + +/** + * There are two testing pages, the current one is described by this `page` + * property: + * - "previous": This is a page testing the previous build with which we want + * to compare to. + * - "current": This is a page testing the current build which we want to + * compare. + */ +let page; +if (location.pathname === "/previous.html") { + page = "previous"; +} else if (location.pathname === "/current.html") { + page = "current"; +} else { + error("Unknown launched page: " + location.pathname); + throw new Error("The current page should either be `previous.html` or `current.html`"); +} + +/** + * Declare a group of tests in a callback that will be performed together and on + * which a timeout may be applied. + * @param {string} name - Name describing this test group + * @param {Function} code - Code implementing that test group. May return a + * promise for asynchronous code. + * @param {number} [timeout] - Optional timeout in milliseconds after which tests + * will be aborted if the test group is not yet finished. + */ +export function declareTestGroup(name, code, timeout) { + if (areTestsAlreadyRunning) { + error(`"declareTestGroup" function call not performed at top level.`); + return; + } + groups.push({ name, code, timeout }); +} + +/** + * Start measuring time for a specific test case. + * Call `testEnd` once done. + * @param {string} testName - The name of the test case (e.g. "seeking"). + */ +export function testStart(name) { + pendingTests.set(name, performance.now()); +} + +/** + * End measuring time for a specific test case started with `testStart`. + * @param {string} testName - The name of the test case (e.g. "seeking"). + */ +export function testEnd(name) { + const startTime = pendingTests.get(name); + if (startTime === undefined) { + error("ERROR: `testEnd` called for inexistant test:", name); + return; + } + pendingTests.delete(name); + reportResult(name, performance.now() - startTime); +} + +/** + * Send log so it's displayed on the Node.js process running those tests. + * @param {Array.} ...logs + */ +export function log(...logs) { + fetch(`http://127.0.0.1:${resultServerPort}`, { + headers: { "Content-Type": "application/json" }, + method: "POST", + body: JSON.stringify({ type: "log", data: logs.join(" ") }), + }).catch((err) => { + // eslint-disable-next-line no-console + console.error("Error: Cannot send log due to a request error.", err); + }); +} + +/** + * Send error interrupting all tests. + * @param {Array.} ...logs + */ +export function error(...logs) { + fetch(`http://127.0.0.1:${resultServerPort}`, { + headers: { "Content-Type": "application/json" }, + method: "POST", + body: JSON.stringify({ type: "error", data: logs.join(" ") }), + }).catch((err) => { + // eslint-disable-next-line no-console + console.error("Error: Cannot send error due to a request error.", err); + }); +} + +/** + * All `declareTestGroup` calls should be done at file evaluation, so we could + * just schedule a micro-task running them when done. + * + * We wait a little more just in case the current page is not following exactly + * that principle. + */ +setTimeout(async () => { + areTestsAlreadyRunning = true; + if (groups.length === 0) { + log("ERROR: No test group declared"); + return; + } + + for (const group of groups) { + const { name, code, timeout } = group; + try { + const res = code(); + if (typeof res === "object" && res !== null && typeof res.then === "function") { + if (typeof timeout === "number") { + await new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + reject(new Error(`Timeout of ${timeout} ms exceeded.`)); + }, timeout); + res.then( + () => { + clearTimeout(timeoutId); + resolve(); + }, + (err) => { + clearTimeout(timeoutId); + reject(err); + }, + ); + }); + } else { + await res; + } + } + } catch (err) { + error("Test group", `"${name}"`, "failed with error:", err.toString()); + return; + } + } + done(); +}, 200); + +/** + * Send results for a specific test case. + * @param {string} testName - The name of the test case (e.g. "seeking"). + * @param {number} result - The time in milliseconds it took to achieve that + * test. + */ +function reportResult(testName, testResult) { + fetch(`http://127.0.0.1:${resultServerPort}`, { + headers: { "Content-Type": "application/json" }, + method: "POST", + body: JSON.stringify({ + type: "value", + page, + data: { name: testName, value: testResult }, + }), + }).catch((err) => { + log("ERROR: Failed to send results for ", testName, err.toString()); + }); +} + +/** + * Called internally once all tests on the page have been performed. Reload the + * page or indicates to the server that it's finished if it is. + */ +function done() { + if (tryAttempt < 100) { + hashComponents.t = tryAttempt + 1; + updateUrlHash(hashComponents); + if (page === "previous") { + location.pathname = "/current.html"; + } else { + location.pathname = "/previous.html"; + } + } else { + sendDone(); + } +} + +/** + * Send internally once tests on that page have been performed enough time. + * Allows the server to close the current browser instance and compile results. + */ +function sendDone() { + fetch(`http://127.0.0.1:${resultServerPort}`, { + headers: { "Content-Type": "application/json" }, + method: "POST", + body: JSON.stringify({ type: "done" }), + }); +} + +/** + * Parse the current URL fragment (format: "#prop1=value1;prop2=value2") and + * return it into a JS object (e.g. `{ prop1: "value1", prop2: "value2" }`) + * etc. + * @returns {Object} + */ +function parseUrlHash() { + const hash = location.hash[0] === "#" ? location.hash.substring(1) : location.hash; + const hashParts = hash.split(";"); + if (hashParts.length === 0) { + throw new Error("The current page should have a fragment present in its URL"); + } + const ret = {}; + for (const hashPart of hashParts) { + const eqlIdx = hashPart.indexOf("="); + if (eqlIdx > 0) { + const propName = hashPart.substring(0, eqlIdx); + ret[propName] = hashPart.substring(eqlIdx + 1); + } + } + return ret; +} + +/** + * Reverse of `parseUrlHash`: take a wanted JS Object (e.g. + * `{ prop1: "value1", prop2: "value2" }`) and put it in the URL's fragment + * so that `parseUrlHash` can then parse it (e.g. "#prop1=value1;prop2=value2"). + * @param {Object} + */ +function updateUrlHash(props) { + let hash = "#"; + for (const prop of Object.keys(props)) { + hash += `${prop}=${props[prop] ?? ""};`; + } + location.hash = hash; +} diff --git a/tests/performance/src/main.js b/tests/performance/src/main.js index 3dedd8eaa1..b095bb01dc 100644 --- a/tests/performance/src/main.js +++ b/tests/performance/src/main.js @@ -1,84 +1,126 @@ import RxPlayer from "rx-player"; -import { manifestInfos } from "../../contents/DASH_static_SegmentTimeline"; +import { MULTI_THREAD } from "rx-player/experimental/features"; +import { EMBEDDED_WORKER } from "rx-player/experimental/features/embeds"; +import { multiAdaptationSetsInfos } from "../../contents/DASH_static_SegmentTimeline"; import sleep from "../../utils/sleep"; import waitForPlayerState, { waitForLoadedStateAfterLoadVideo, } from "../../utils/waitForPlayerState"; +import { declareTestGroup, testEnd, testStart } from "./lib"; -let player; - -test(); - -async function test() { - await sleep(200); - const timeBeforeLoad = performance.now(); - player = new RxPlayer({ - initialVideoBitrate: Infinity, - initialAudioBitrate: Infinity, - videoElement: document.getElementsByTagName("video")[0], - }); - player.loadVideo({ - url: manifestInfos.url, - transport: manifestInfos.transport, - }); - await waitForLoadedStateAfterLoadVideo(player); - const timeToLoad = performance.now() - timeBeforeLoad; - sendTestResult("loading", timeToLoad); - await sleep(1); - const timeBeforeSeek = performance.now(); - player.seekTo(20); - await waitForPlayerState(player, "PAUSED", ["SEEKING", "BUFFERING"]); - const timeToSeek = performance.now() - timeBeforeSeek; - sendTestResult("seeking", timeToSeek); - reloadIfNeeded(); -} - -function sendTestResult(testName, testResult) { - fetch("http://127.0.0.1:6789", { - headers: { "Content-Type": "application/json" }, - method: "POST", - body: JSON.stringify({ - type: "value", - data: { name: testName, value: testResult }, - }), - }); -} - -function sendLog(log) { - fetch("http://127.0.0.1:6789", { - headers: { "Content-Type": "application/json" }, - method: "POST", - body: JSON.stringify({ type: "log", data: log }), - }).catch((err) => { - // eslint-disable-next-line no-console - console.error("Error: Cannot send log due to a request error.", err); - }); -} - -function reloadIfNeeded() { - const testNumber = getTestNumber(); - if (testNumber < 100) { - location.hash = "#" + (testNumber + 1); - location.reload(); - } else { - sendDone(); - } -} - -function sendDone() { - fetch("http://127.0.0.1:6789", { - headers: { "Content-Type": "application/json" }, - method: "POST", - body: JSON.stringify({ type: "done" }), - }); -} - -function getTestNumber() { - if (location.hash === "") { - return 1; - } - return Number(location.hash.substring(1)); -} - -// Allow to display logs in the RxPlayer source code -window.sendLog = sendLog; +declareTestGroup( + "content loading monothread", + async () => { + // --- 1: load --- + + testStart("loading"); + const player = new RxPlayer({ + initialVideoBitrate: Infinity, + initialAudioBitrate: Infinity, + videoElement: document.getElementsByTagName("video")[0], + }); + player.loadVideo({ + url: multiAdaptationSetsInfos.url, + transport: multiAdaptationSetsInfos.transport, + }); + await waitForLoadedStateAfterLoadVideo(player); + testEnd("loading"); + await sleep(10); + + // --- 2: seek --- + + testStart("seeking"); + player.seekTo(20); + await waitForPlayerState(player, "PAUSED", ["SEEKING", "BUFFERING"]); + testEnd("seeking"); + await sleep(10); + + // -- 3: change audio track + reload --- + + testStart("audio-track-reload"); + const audioTracks = player.getAvailableAudioTracks(); + if (audioTracks.length < 2) { + throw new Error("Not enough audio tracks for audio track switching"); + } + + for (const audioTrack of audioTracks) { + if (!audioTrack.active) { + player.setAudioTrack({ trackId: audioTrack.id, switchingMode: "reload" }); + } + } + await waitForPlayerState(player, "PAUSED"); + testEnd("audio-track-reload"); + + player.dispose(); + await sleep(10); // ensure dispose is done + }, + 20000, +); + +declareTestGroup( + "content loading multithread", + async () => { + // --- 1: cold loading (Worker attachment etc.) --- + + testStart("cold loading multithread"); + const player = new RxPlayer({ + initialVideoBitrate: Infinity, + initialAudioBitrate: Infinity, + videoElement: document.getElementsByTagName("video")[0], + }); + RxPlayer.addFeatures([MULTI_THREAD]); + player.attachWorker({ + workerUrl: EMBEDDED_WORKER, + }); + player.loadVideo({ + url: multiAdaptationSetsInfos.url, + transport: multiAdaptationSetsInfos.transport, + mode: "multithread", + }); + await waitForLoadedStateAfterLoadVideo(player); + testEnd("cold loading multithread"); + await sleep(10); + + // --- 2: seek --- + + testStart("seeking multithread"); + player.seekTo(20); + await waitForPlayerState(player, "PAUSED", ["SEEKING", "BUFFERING"]); + testEnd("seeking multithread"); + await sleep(10); + + // -- 3: change audio track + reload --- + + testStart("audio-track-reload multithread"); + const audioTracks = player.getAvailableAudioTracks(); + if (audioTracks.length < 2) { + throw new Error("Not enough audio tracks for audio track switching"); + } + + for (const audioTrack of audioTracks) { + if (!audioTrack.active) { + player.setAudioTrack({ trackId: audioTrack.id, switchingMode: "reload" }); + } + } + await waitForPlayerState(player, "PAUSED"); + testEnd("audio-track-reload multithread"); + + player.stop(); + + // --- 4: hot loading --- + + await sleep(10); + testStart("hot loading multithread"); + player.loadVideo({ + url: multiAdaptationSetsInfos.url, + transport: multiAdaptationSetsInfos.transport, + mode: "multithread", + }); + await waitForLoadedStateAfterLoadVideo(player); + testEnd("hot loading multithread"); + + player.dispose(); + await sleep(10); // ensure dispose is done + }, + 20000, +);