Add CV convergence logic

src/benchmark/benchmarkFn.ts

@@ -25,20 +25,20 @@ export const bench: BenchApi = createBenchmarkFunction(function <T, T2>(
       throw Error(`test titles must be unique, duplicated: '${opts.id}'`);
     }
 
-    // Persist full results if requested. dir is created in `beforeAll`
-    const benchmarkResultsCsvDir = process.env.BENCHMARK_RESULTS_CSV_DIR;
-    const persistRunsNs = Boolean(benchmarkResultsCsvDir);
-
-    const {result, runsNs} = await runBenchFn<T, T2>(
-      {...options, fn: benchTask, before, beforeEach} as BenchmarkRunOptsWithFn<T, T2>,
-      persistRunsNs
-    );
+    const {result, runsNs} = await runBenchFn<T, T2>({
+      ...options,
+      fn: benchTask,
+      before,
+      beforeEach,
+    } as BenchmarkRunOptsWithFn<T, T2>);
 
     // Store result for:
     // - to persist benchmark data latter
     // - to render with the custom reporter
     store.setResult(opts.id, result);
 
+    // Persist full results if requested. dir is created in `beforeAll`
+    const benchmarkResultsCsvDir = process.env.BENCHMARK_RESULTS_CSV_DIR;
     if (benchmarkResultsCsvDir) {
       fs.mkdirSync(benchmarkResultsCsvDir, {recursive: true});
       const filename = `${result.id}.csv`;

src/benchmark/runBenchmarkFn.ts

@@ -1,6 +1,6 @@
 import {BenchmarkResult, BenchmarkOpts} from "../types.js";
 import {getBenchmarkOptionsWithDefaults} from "./options.js";
-import {createConvergenceCriteria} from "./termination.js";
+import {createCVConvergenceCriteria} from "./termination.js";
 
 export type BenchmarkRunOpts = BenchmarkOpts & {
   id: string;
@@ -14,8 +14,7 @@ export type BenchmarkRunOptsWithFn<T, T2> = BenchmarkOpts & {
 };
 
 export async function runBenchFn<T, T2>(
-  opts: BenchmarkRunOptsWithFn<T, T2>,
-  persistRunsNs?: boolean
+  opts: BenchmarkRunOptsWithFn<T, T2>
 ): Promise<{result: BenchmarkResult; runsNs: bigint[]}> {
   const {id, before, beforeEach, fn, ...rest} = opts;
   const benchOptions = getBenchmarkOptionsWithDefaults(rest);
@@ -36,7 +35,7 @@ export async function runBenchFn<T, T2>(
   const runsNs: bigint[] = [];
   const startRunMs = Date.now();
 
-  const shouldTerminate = createConvergenceCriteria(startRunMs, benchOptions);
+  const shouldTerminate = createCVConvergenceCriteria(startRunMs, benchOptions);
 
   let runIdx = 0;
   let totalNs = BigInt(0);
@@ -79,13 +78,11 @@ export async function runBenchFn<T, T2>(
     // Persist results
     runIdx += 1;
     totalNs += runNs;
+    runsNs.push(runNs);
 
-    if (shouldTerminate(runIdx, totalNs)) {
+    if (shouldTerminate(runIdx, totalNs, runsNs)) {
       break;
     }
-
-    // If the caller wants the exact times of all runs, persist them
-    if (persistRunsNs) runsNs.push(runNs);
   }
 
   if (runIdx === 0) {

src/benchmark/termination.ts

@@ -1,8 +1,9 @@
 import {BenchmarkOpts} from "../types.js";
+import {calcMean, calcMedian, calcVariance, filterOutliers, OutlierSensitivity, sortData} from "../utils/math.js";
 
-export type TerminationCriteria = (runIdx: number, totalNs: bigint) => boolean;
+export type TerminationCriteria = (runIdx: number, totalNs: bigint, runNs: bigint[]) => boolean;
 
-export function createConvergenceCriteria(
+export function createLinearConvergenceCriteria(
   startMs: number,
   {maxMs, maxRuns, minRuns, minMs, convergeFactor}: Required<BenchmarkOpts>
 ): TerminationCriteria {
@@ -11,7 +12,8 @@ export function createConvergenceCriteria(
   let lastConvergenceSample = startMs;
   const sampleEveryMs = 100;
 
-  return function canTerminate(runIdx: number, totalNs: bigint): boolean {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  return function canTerminate(runIdx: number, totalNs: bigint, _runNs: bigint[]): boolean {
     const currentMs = Date.now();
     const elapsedMs = currentMs - startMs;
     const mustStop = elapsedMs >= maxMs || runIdx >= maxRuns;
@@ -52,3 +54,48 @@ export function createConvergenceCriteria(
     return false;
   };
 }
+
+export function createCVConvergenceCriteria(
+  startMs: number,
+  {maxMs, maxRuns, minRuns, minMs, convergeFactor}: Required<BenchmarkOpts>
+): TerminationCriteria {
+  let lastConvergenceSample = startMs;
+  const sampleEveryMs = 100;
+  const minSamples = minRuns > 5 ? minRuns : 5;
+  const maxSamplesForCV = 1000;
+
+  return function canTerminate(runIdx: number, totalNs: bigint, runsNs: bigint[]): boolean {
+    const currentMs = Date.now();
+    const elapsedMs = currentMs - startMs;
+    const mustStop = elapsedMs >= maxMs || runIdx >= maxRuns;
+    const mayStop = elapsedMs >= minMs && runIdx >= minRuns && runIdx > minSamples;
+
+    // Must stop
+    if (mustStop) return true;
+
+    if (Date.now() - lastConvergenceSample <= sampleEveryMs) return false;
+
+    if (mayStop) {
+      lastConvergenceSample = currentMs;
+
+      const mean = calcMean(runsNs);
+      const variance = calcVariance(runsNs, mean);
+      const cv = Math.sqrt(Number(variance)) / Number(mean);
+
+      if (cv < convergeFactor) return true;
+
+      // If CV does not stabilize we fallback to the median approach
+      if (runsNs.length > maxSamplesForCV) {
+        const sorted = sortData(runsNs);
+        const cleanedRunsNs = filterOutliers(sorted, true, OutlierSensitivity.Mild);
+        const median = calcMedian(cleanedRunsNs, true);
+        const mean = calcMean(cleanedRunsNs);
+        const medianFactor = Math.abs(Number(mean - median)) / Number(median);
+
+        if (medianFactor < convergeFactor) return true;
+      }
+    }
+
+    return false;
+  };
+}

src/utils/math.ts

@@ -0,0 +1,77 @@
+export function calcSum(arr: bigint[]): bigint {
+  let s = BigInt(0);
+
+  for (const n of arr) {
+    s += n;
+  }
+  return s;
+}
+
+export function calcMean(arr: bigint[]): bigint {
+  return BigInt(calcSum(arr) / BigInt(arr.length));
+}
+
+export function calcVariance(arr: bigint[], mean: bigint): bigint {
+  let base = BigInt(0);
+
+  for (const n of arr) {
+    const diff = n - mean;
+    base += diff * diff;
+  }
+
+  return base / BigInt(arr.length);
+}
+
+export function sortData(arr: bigint[]): bigint[] {
+  return [...arr].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
+}
+
+export function calcMedian(arr: bigint[], sorted: boolean): bigint {
+  // 1. Sort the BigInt array
+  const data = sorted ? arr : sortData(arr);
+
+  // 3. Calculate median
+  const mid = Math.floor(data.length / 2);
+  if (data.length % 2 === 0) {
+    return (data[mid - 1] + data[mid]) / BigInt(2); // Average two middle values
+  } else {
+    return data[mid]; // Single middle value
+  }
+}
+
+export function calcQuartile(sortedData: bigint[], percentile: number): bigint {
+  const index = (sortedData.length - 1) * percentile;
+  const floor = Math.floor(index);
+  const fraction = index - floor;
+
+  if (sortedData[floor + 1] !== undefined) {
+    return BigInt(Number(sortedData[floor]) + fraction * Number(sortedData[floor + 1] - sortedData[floor]));
+  } else {
+    return sortedData[floor];
+  }
+}
+
+export enum OutlierSensitivity {
+  Mild = 1.5,
+  Strict = 3.0,
+}
+
+export function filterOutliers(arr: bigint[], sorted: boolean, sensitivity: OutlierSensitivity): bigint[] {
+  if (arr.length < 4) return arr; // Too few data points
+
+  const data = sorted ? arr : sortData(arr);
+
+  // Calculate quartiles and IQR
+  const q1 = Number(calcQuartile(data, 0.25));
+  const q3 = Number(calcQuartile(data, 0.75));
+  const iqr = q3 - q1;
+
+  // Define outlier bounds (adjust multiplier for sensitivity)
+  const lowerBound = q1 - sensitivity * iqr;
+  const upperBound = q3 + sensitivity * iqr;
+
+  // Filter original BigInt values
+  return data.filter((n) => {
+    return n >= lowerBound && n <= upperBound;
+  });
+}

test/perf/iteration.test.ts

@@ -8,7 +8,7 @@ import {bench, describe, setBenchOpts} from "../../src/index.js";
 // byteArrayEquals with valueOf()                                         853971.0 ops/s      1.171000 us/op 9963051 runs    16.07 s
 
 describe("Array iteration", () => {
-  setBenchOpts({maxMs: 60 * 1000, convergeFactor: 0.1 / 100});
+  setBenchOpts({maxMs: 60 * 1000, convergeFactor: 1 / 100});
 
   // nonce = 5
   const n = 1e6;