From 7628043a6fc6ba898301915bb1917ff02bb04817 Mon Sep 17 00:00:00 2001 From: Dom Slee Date: Tue, 19 Mar 2024 00:18:04 +1100 Subject: [PATCH] Achieve better results with compiler flags, use median for benchmarks (#22) --- Cargo.toml | 1 + benchmark/all.md | 28 +++++++++++------------ benchmark/complete/benchmark_complete.ps1 | 4 ++-- benchmark/init/benchmark_init.ps1 | 2 +- benchmark/util.ps1 | 6 ++++- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 40b5437..1c80daf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,3 +38,4 @@ lto = true strip = true opt-level = "z" codegen-units = 1 +panic = "abort" diff --git a/benchmark/all.md b/benchmark/all.md index 54e7318..4ce79e1 100644 --- a/benchmark/all.md +++ b/benchmark/all.md @@ -1,22 +1,22 @@ # All results -file size: 13297664 -pwsh version: 7.4.0-rc.1 +file size: 9488384 +pwsh version: 7.5.0-preview.2 ## Init -| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | -| :----------------------------------------------------------- | ----------: | -------: | -------: | ----------: | -| `pwsh -NoProfile -File ./../profiles/ProfileBaseline.ps1` | 207.6 ± 6.7 | 186.8 | 233.2 | 1.00 | -| `pwsh -NoProfile -File ./../profiles/ProfilePoshGit.ps1` | 487.7 ± 8.0 | 478.1 | 524.3 | 2.35 ± 0.08 | -| `pwsh -NoProfile -File ./../profiles/ProfileTabComplete.ps1` | 253.4 ± 6.4 | 247.2 | 284.2 | 1.22 ± 0.05 | +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +|:---|---:|---:|---:|---:| +| `pwsh -NoProfile -File ./../profiles/ProfileBaseline.ps1` | 204.9 ± 3.8 | 201.7 | 220.4 | 1.00 | +| `pwsh -NoProfile -File ./../profiles/ProfilePoshGit.ps1` | 473.5 ± 5.1 | 466.0 | 499.5 | 2.31 ± 0.05 | +| `pwsh -NoProfile -File ./../profiles/ProfileTabComplete.ps1` | 246.5 ± 2.5 | 242.6 | 262.1 | 1.20 ± 0.03 | -posh-tabcomplete: 45ms, posh-git: 280ms (6.22x faster) +posh-tabcomplete: 41ms, posh-git: 268ms (6.54x faster) ## Complete -| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | -| :---------------------------------------------- | -----------: | -------: | -------: | ----------: | -| `pwsh -NoProfile -File CompleteBaseline.ps1` | 523.9 ± 11.8 | 510.4 | 584.0 | 1.00 | -| `pwsh -NoProfile -File CompletePoshGit.ps1` | 746.5 ± 28.0 | 721.4 | 843.8 | 1.42 ± 0.06 | -| `pwsh -NoProfile -File CompleteTabComplete.ps1` | 628.0 ± 19.5 | 609.1 | 736.2 | 1.20 ± 0.05 | +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +|:---|---:|---:|---:|---:| +| `pwsh -NoProfile -File CompleteBaseline.ps1` | 503.4 ± 7.1 | 495.3 | 565.0 | 1.00 | +| `pwsh -NoProfile -File CompletePoshGit.ps1` | 683.2 ± 5.7 | 675.8 | 708.0 | 1.36 ± 0.02 | +| `pwsh -NoProfile -File CompleteTabComplete.ps1` | 586.7 ± 4.3 | 580.7 | 607.7 | 1.17 ± 0.02 | -posh-tabcomplete: 104ms, posh-git: 222ms (2.13x faster) +posh-tabcomplete: 83ms, posh-git: 179ms (2.16x faster) diff --git a/benchmark/complete/benchmark_complete.ps1 b/benchmark/complete/benchmark_complete.ps1 index aee7bd9..b0ecd40 100644 --- a/benchmark/complete/benchmark_complete.ps1 +++ b/benchmark/complete/benchmark_complete.ps1 @@ -1,5 +1,6 @@ . $PSScriptRoot/../setupErrorHandling.ps1 +. $PSScriptRoot/../util.ps1 $childRepoDir = "$PSScriptRoot/repo" if (Test-Path "$childRepoDir") { Remove-Item -r -fo "$childRepoDir" @@ -19,13 +20,12 @@ git commit -m "init" Set-Location "$PSScriptRoot" hyperfine ` --warmup 3 ` - --runs 75 ` + --runs (GetNumRuns) ` -L script CompleteBaseline.ps1,CompletePoshGit.ps1,CompleteTabComplete.ps1 ` "pwsh -NoProfile -File {script}" ` --export-markdown $PSScriptRoot/complete.md ` --export-csv $PSScriptRoot/complete.csv -. $PSScriptRoot/../util.ps1 $csv = "$PSScriptRoot/complete.csv" $tabMs = GetMs $csv "CompleteBaseline.ps1" "CompleteTabComplete.ps1"; $poshMs = GetMs $csv "CompleteBaseline.ps1" "CompletePoshGit.ps1"; diff --git a/benchmark/init/benchmark_init.ps1 b/benchmark/init/benchmark_init.ps1 index f5cf217..9f26868 100644 --- a/benchmark/init/benchmark_init.ps1 +++ b/benchmark/init/benchmark_init.ps1 @@ -3,7 +3,7 @@ Set-Location $PSScriptRoot hyperfine ` --warmup 3 ` - --runs 75 ` + --runs (GetNumRuns) ` -L profile ./../profiles/ProfileBaseline.ps1,./../profiles/ProfilePoshGit.ps1,./../profiles/ProfileTabComplete.ps1 ` "pwsh -NoProfile -File {profile}" ` --export-markdown init.md ` diff --git a/benchmark/util.ps1 b/benchmark/util.ps1 index 4251b01..1c5628a 100644 --- a/benchmark/util.ps1 +++ b/benchmark/util.ps1 @@ -2,11 +2,15 @@ function GetMs { param ([string] $csv, [string] $baselineFile, [string] $profileFile) $baselineEntry = Import-Csv $csv | Where-Object { $_.command.Contains($baselineFile) } | Select-Object -First 1 $entry = Import-Csv $csv | Where-Object { $_.command.Contains($profileFile) } | Select-Object -First 1 - $seconds = $entry.mean - $baselineEntry.mean + $seconds = $entry.median - $baselineEntry.median return [math]::Round([timespan]::fromseconds($seconds).Milliseconds, 2) } function GetSummary { param ([double] $poshMs, [double] $tabMs) return "posh-tabcomplete: ${tabMs}ms, posh-git: ${poshMs}ms ($([math]::Round($poshMs / $tabMs, 2))x faster)" +} + +function GetNumRuns { + return 150; } \ No newline at end of file