Skip to content

Commit

Permalink
clean up
Browse files Browse the repository at this point in the history
  • Loading branch information
bdemann committed Dec 17, 2024
1 parent e507126 commit 133a4be
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 27 deletions.
7 changes: 5 additions & 2 deletions scripts/analyze_benchmarks/markdown.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,10 @@ ${Object.entries(results.stable)
}

function formatChangeValue(change: number): string {
if (change === 0) return `${change.toFixed(2)}%`;
if (change === 0) {
return `${change.toFixed(2)}%`;
}

return `<span style="color: ${change < 0 ? 'green' : 'red'}">${change.toFixed(2)}%</span>`;
}

Expand All @@ -161,7 +164,7 @@ function compareChanges(
);
return { stable, experimental };
}
// Return a Statistics object with all values set to 0 if there's no previous version

return {
stable: createDefaultStatistics(),
experimental: createDefaultStatistics()
Expand Down
65 changes: 40 additions & 25 deletions scripts/analyze_benchmarks/statistics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ export type Statistics = {
baselineWeightedEfficiencyScore: number;
};

const MAX_VALID_INSTRUCTIONS = 40_000_000_000;
const EFFICIENCY_WEIGHTS = {
min: 0.6,
median: 0.3,
mean: 0.1
} as const;

export function calculateVersionStatistics(
entries: BenchmarkEntry[]
): Statistics {
Expand All @@ -27,7 +34,7 @@ function calculateStatistics(instructions: number[]): Statistics {

// Filter out instructions > 40 billion
const filteredInstructions = instructions.filter(
(val) => val <= 40_000_000_000
(val) => val <= MAX_VALID_INSTRUCTIONS
);

if (filteredInstructions.length === 0) {
Expand All @@ -37,27 +44,15 @@ function calculateStatistics(instructions: number[]): Statistics {
}

const sorted = [...filteredInstructions].sort((a, b) => a - b);
const mean =
filteredInstructions.reduce((acc, val) => acc + val, 0) /
filteredInstructions.length;

const mid = Math.floor(sorted.length / 2);
const median =
sorted.length % 2 === 0
? (sorted[mid - 1] + sorted[mid]) / 2
: sorted[mid];

const standardDeviation = Math.sqrt(
filteredInstructions
.map((value) => Math.pow(value - mean, 2))
.reduce((acc, val) => acc + val, 0) / filteredInstructions.length
);

const count = filteredInstructions.length;
const min = sorted[0];
const median = calculateMedian(sorted);
const mean = calculateMean(filteredInstructions);
const standardDeviation = calculateStandardDeviation(filteredInstructions);
const max = sorted[sorted.length - 1];
const baselineWeightedEfficiencyScore =
calculateBaselineWeightEfficiencyScores(min, median, mean);
calculateBaselineWeightEfficiencyScore(min, median, mean);

return {
count,
Expand All @@ -70,16 +65,36 @@ function calculateStatistics(instructions: number[]): Statistics {
};
}

function calculateBaselineWeightEfficiencyScores(
function calculateMean(instructions: readonly number[]): number {
return (
instructions.reduce((acc, val) => acc + val, 0) / instructions.length
);
}

function calculateMedian(sorted: readonly number[]): number {
const mid = Math.floor(sorted.length / 2);
return sorted.length % 2 === 0
? (sorted[mid - 1] + sorted[mid]) / 2
: sorted[mid];
}

function calculateStandardDeviation(instructions: readonly number[]): number {
const mean = calculateMean(instructions);
return Math.sqrt(
instructions
.map((value) => Math.pow(value - mean, 2))
.reduce((acc, val) => acc + val, 0) / instructions.length
);
}

function calculateBaselineWeightEfficiencyScore(
min: number,
median: number,
mean: number
): number {
const weights = {
min: 0.6,
median: 0.3,
mean: 0.1
} as const;

return weights.min * min + weights.median * median + weights.mean * mean;
return (
EFFICIENCY_WEIGHTS.min * min +
EFFICIENCY_WEIGHTS.median * median +
EFFICIENCY_WEIGHTS.mean * mean
);
}

0 comments on commit 133a4be

Please sign in to comment.