Improve performance tests writing

This is an attempt at facilitating the writing of our performance tests, which measure the time taken by key actions and whether we had performance regressions for them (e.g. longer to load a content). The idea is just that I move client-side test management code to a `lib.js` file, and now original test files only contains steps and take the more declarative format we're used with other kind of tests. `lib.js` is then the one running them, sending results, handling errors etc. I also profited from this to add performance tests for multithread content loading, track switching and reloading.
canalplus · Jan 14, 2025 · 3ea979e · 3ea979e
1 parent b56ba61
commit 3ea979e
Show file tree

Hide file tree

Showing 3 changed files with 295 additions and 79 deletions.
diff --git a/tests/performance/run.mjs b/tests/performance/run.mjs
@@ -455,7 +455,11 @@ function createResultServer() {
           const parsedBody = JSON.parse(body);
           if (parsedBody.type === "log") {
             // eslint-disable-next-line no-console
-            console.log("LOG:", parsedBody.data);
+            console.warn("LOG:", parsedBody.data);
+          } else if (parsedBody.type === "error") {
+            // eslint-disable-next-line no-console
+            console.error("ERROR: A fatal error happened:", parsedBody.data);
+            process.exit(1);
           } else if (parsedBody.type === "done") {
             if (currentBrowser !== undefined) {
               currentBrowser.kill();

diff --git a/tests/performance/src/lib.js b/tests/performance/src/lib.js
@@ -0,0 +1,170 @@
+const pendingTests = new Map();
+const groups = [];
+let areTestsAlreadyRunning = false;
+
+/**
+ * Declare a group of tests in a callback that will be performed together and on
+ * which a timeout may be applied.
+ * @param {string} name - Name describing this test group
+ * @param {Function} code - Code implementing that test group. May return a
+ * promise for asynchronous code.
+ * @param {number} [timeout] - Optional timeout in milliseconds after which tests
+ * will be aborted if the test group is not yet finished.
+ */
+export function declareTestGroup(name, code, timeout) {
+  if (areTestsAlreadyRunning) {
+    error(`"declareTestGroup" function called not performed at top level.`);
+    return;
+  }
+  groups.push({ name, code, timeout });
+}
+
+/**
+ * Start measuring time for a specific test case.
+ * Call `testEnd` once done.
+ * @param {string} testName - The name of the test case (e.g. "seeking").
+ */
+export function testStart(name) {
+  pendingTests.set(name, performance.now());
+}
+
+/**
+ * End measuring time for a specific test case started with `testStart`.
+ * @param {string} testName - The name of the test case (e.g. "seeking").
+ */
+export function testEnd(name) {
+  const startTime = pendingTests.get(name);
+  if (startTime === undefined) {
+    error("ERROR: `testEnd` called for inexistant test:", name);
+    return;
+  }
+  reportResult(name, performance.now() - startTime);
+}
+
+/**
+ * Send log so it's displayed on the Node.js process running those tests.
+ * @param {Array.<string>} ...logs
+ */
+export function log(...logs) {
+  fetch("http://127.0.0.1:6789", {
+    headers: { "Content-Type": "application/json" },
+    method: "POST",
+    body: JSON.stringify({ type: "log", data: logs.join(" ") }),
+  }).catch((err) => {
+    // eslint-disable-next-line no-console
+    console.error("Error: Cannot send log due to a request error.", err);
+  });
+}
+
+/**
+ * Send error interrupting all tests.
+ * @param {Array.<string>} ...logs
+ */
+export function error(...logs) {
+  fetch("http://127.0.0.1:6789", {
+    headers: { "Content-Type": "application/json" },
+    method: "POST",
+    body: JSON.stringify({ type: "error", data: logs.join(" ") }),
+  }).catch((err) => {
+    // eslint-disable-next-line no-console
+    console.error("Error: Cannot send error due to a request error.", err);
+  });
+}
+
+/**
+ * All `declareTestGroup` calls should be done at file evaluation, so we could
+ * just schedule a micro-task running them when done.
+ *
+ * We wait a little more just to be sure that the page is completely loaded.
+ */
+setTimeout(async () => {
+  areTestsAlreadyRunning = true;
+  if (groups.length === 0) {
+    log("ERROR: No test group declared");
+    return;
+  }
+
+  for (const group of groups) {
+    const { name, code, timeout } = group;
+    try {
+      const res = code();
+      if (typeof res === "object" && res !== null && typeof res.then === "function") {
+        if (typeof timeout === "number") {
+          await new Promise((resolve, reject) => {
+            const timeoutId = setTimeout(() => {
+              reject(new Error(`Timeout of ${timeout} ms exceeded.`));
+            }, timeout);
+            res.then(
+              () => {
+                clearTimeout(timeoutId);
+                resolve();
+              },
+              (err) => {
+                clearTimeout(timeoutId);
+                reject(err);
+              },
+            );
+          });
+        } else {
+          await res;
+        }
+      }
+    } catch (err) {
+      error("Test group", `"${name}"`, "failed with error:", err.toString());
+      return;
+    }
+  }
+  done();
+}, 200);
+
+/**
+ * Send results for a specific test case.
+ * @param {string} testName - The name of the test case (e.g. "seeking").
+ * @param {number} result - The time in milliseconds it took to achieve that
+ * test.
+ */
+function reportResult(testName, testResult) {
+  fetch("http://127.0.0.1:6789", {
+    headers: { "Content-Type": "application/json" },
+    method: "POST",
+    body: JSON.stringify({
+      type: "value",
+      data: { name: testName, value: testResult },
+    }),
+  }).catch((err) => {
+    log("ERROR: Failed to send results for ", testName, err.toString());
+  });
+}
+
+/**
+ * Called internally once all tests on the page have been performed. Reload the
+ * page or indicates to the server that it's finished if it is.
+ */
+function done() {
+  const testNumber = getTestNumber();
+  if (testNumber < 100) {
+    location.hash = "#" + (testNumber + 1);
+    location.reload();
+  } else {
+    sendDone();
+  }
+}
+
+function getTestNumber() {
+  if (location.hash === "") {
+    return 1;
+  }
+  return Number(location.hash.substring(1));
+}
+
+/**
+ * Send internally once tests on that page have been performed enough time.
+ * Allows the server to close the current browser instance and compile results.
+ */
+function sendDone() {
+  fetch("http://127.0.0.1:6789", {
+    headers: { "Content-Type": "application/json" },
+    method: "POST",
+    body: JSON.stringify({ type: "done" }),
+  });
+}
diff --git a/tests/performance/src/main.js b/tests/performance/src/main.js
@@ -1,84 +1,126 @@
 import RxPlayer from "rx-player";
-import { manifestInfos } from "../../contents/DASH_static_SegmentTimeline";
+import { MULTI_THREAD } from "rx-player/experimental/features";
+import { EMBEDDED_WORKER } from "rx-player/experimental/features/embeds";
+import { multiAdaptationSetsInfos } from "../../contents/DASH_static_SegmentTimeline";
 import sleep from "../../utils/sleep";
 import waitForPlayerState, {
   waitForLoadedStateAfterLoadVideo,
 } from "../../utils/waitForPlayerState";
+import { declareTestGroup, testEnd, testStart } from "./lib";
 
-let player;
-
-test();
-
-async function test() {
-  await sleep(200);
-  const timeBeforeLoad = performance.now();
-  player = new RxPlayer({
-    initialVideoBitrate: Infinity,
-    initialAudioBitrate: Infinity,
-    videoElement: document.getElementsByTagName("video")[0],
-  });
-  player.loadVideo({
-    url: manifestInfos.url,
-    transport: manifestInfos.transport,
-  });
-  await waitForLoadedStateAfterLoadVideo(player);
-  const timeToLoad = performance.now() - timeBeforeLoad;
-  sendTestResult("loading", timeToLoad);
-  await sleep(1);
-  const timeBeforeSeek = performance.now();
-  player.seekTo(20);
-  await waitForPlayerState(player, "PAUSED", ["SEEKING", "BUFFERING"]);
-  const timeToSeek = performance.now() - timeBeforeSeek;
-  sendTestResult("seeking", timeToSeek);
-  reloadIfNeeded();
-}
-
-function sendTestResult(testName, testResult) {
-  fetch("http://127.0.0.1:6789", {
-    headers: { "Content-Type": "application/json" },
-    method: "POST",
-    body: JSON.stringify({
-      type: "value",
-      data: { name: testName, value: testResult },
-    }),
-  });
-}
-
-function sendLog(log) {
-  fetch("http://127.0.0.1:6789", {
-    headers: { "Content-Type": "application/json" },
-    method: "POST",
-    body: JSON.stringify({ type: "log", data: log }),
-  }).catch((err) => {
-    // eslint-disable-next-line no-console
-    console.error("Error: Cannot send log due to a request error.", err);
-  });
-}
-
-function reloadIfNeeded() {
-  const testNumber = getTestNumber();
-  if (testNumber < 100) {
-    location.hash = "#" + (testNumber + 1);
-    location.reload();
-  } else {
-    sendDone();
-  }
-}
-
-function sendDone() {
-  fetch("http://127.0.0.1:6789", {
-    headers: { "Content-Type": "application/json" },
-    method: "POST",
-    body: JSON.stringify({ type: "done" }),
-  });
-}
-
-function getTestNumber() {
-  if (location.hash === "") {
-    return 1;
-  }
-  return Number(location.hash.substring(1));
-}
-
-// Allow to display logs in the RxPlayer source code
-window.sendLog = sendLog;
+declareTestGroup(
+  "content loading monothread",
+  async () => {
+    // --- 1: load ---
+
+    testStart("loading");
+    const player = new RxPlayer({
+      initialVideoBitrate: Infinity,
+      initialAudioBitrate: Infinity,
+      videoElement: document.getElementsByTagName("video")[0],
+    });
+    player.loadVideo({
+      url: multiAdaptationSetsInfos.url,
+      transport: multiAdaptationSetsInfos.transport,
+    });
+    await waitForLoadedStateAfterLoadVideo(player);
+    testEnd("loading");
+    await sleep(10);
+
+    // --- 2: seek ---
+
+    testStart("seeking");
+    player.seekTo(20);
+    await waitForPlayerState(player, "PAUSED", ["SEEKING", "BUFFERING"]);
+    testEnd("seeking");
+    await sleep(10);
+
+    // -- 3: change audio track + reload ---
+
+    testStart("audio-track-reload");
+    const audioTracks = player.getAvailableAudioTracks();
+    if (audioTracks.length < 2) {
+      throw new Error("Not enough audio tracks for audio track switching");
+    }
+
+    for (const audioTrack of audioTracks) {
+      if (!audioTrack.active) {
+        player.setAudioTrack({ trackId: audioTrack.id, switchingMode: "reload" });
+      }
+    }
+    await waitForPlayerState(player, "PAUSED");
+    testEnd("audio-track-reload");
+
+    player.dispose();
+    await sleep(10); // ensure dispose is done
+  },
+  10000,
+);
+
+declareTestGroup(
+  "content loading multithread",
+  async () => {
+    // --- 1: cold loading (Worker attachment etc.) ---
+
+    testStart("cold loading multithread");
+    const player = new RxPlayer({
+      initialVideoBitrate: Infinity,
+      initialAudioBitrate: Infinity,
+      videoElement: document.getElementsByTagName("video")[0],
+    });
+    RxPlayer.addFeatures([MULTI_THREAD]);
+    player.attachWorker({
+      workerUrl: EMBEDDED_WORKER,
+    });
+    player.loadVideo({
+      url: multiAdaptationSetsInfos.url,
+      transport: multiAdaptationSetsInfos.transport,
+      mode: "multithread",
+    });
+    await waitForLoadedStateAfterLoadVideo(player);
+    testEnd("cold loading multithread");
+    await sleep(10);
+
+    // --- 2: seek ---
+
+    testStart("seeking multithread");
+    player.seekTo(20);
+    await waitForPlayerState(player, "PAUSED", ["SEEKING", "BUFFERING"]);
+    testEnd("seeking multithread");
+    await sleep(10);
+
+    // -- 3: change audio track + reload ---
+
+    testStart("audio-track-reload multithread");
+    const audioTracks = player.getAvailableAudioTracks();
+    if (audioTracks.length < 2) {
+      throw new Error("Not enough audio tracks for audio track switching");
+    }
+
+    for (const audioTrack of audioTracks) {
+      if (!audioTrack.active) {
+        player.setAudioTrack({ trackId: audioTrack.id, switchingMode: "reload" });
+      }
+    }
+    await waitForPlayerState(player, "PAUSED");
+    testEnd("audio-track-reload multithread");
+
+    player.stop();
+
+    // --- 4: hot loading ---
+
+    await sleep(10);
+    testStart("hot loading multithread");
+    player.loadVideo({
+      url: multiAdaptationSetsInfos.url,
+      transport: multiAdaptationSetsInfos.transport,
+      mode: "multithread",
+    });
+    await waitForLoadedStateAfterLoadVideo(player);
+    testEnd("hot loading multithread");
+
+    player.dispose();
+    await sleep(10); // ensure dispose is done
+  },
+  10000,
+);