From 7a44b557e87cc2099a3bd40ff2ec5a1252bbe978 Mon Sep 17 00:00:00 2001 From: Jacob Colvin Date: Sat, 4 Jan 2025 17:33:56 -0500 Subject: [PATCH] Update benchmarks --- README.md | 2 +- Taskfile.yaml | 29 +++++++++++++++---- ...-helm-values.k => 10-charts-with-values.k} | 7 +++-- benchmarks/{simple-helm.k => 10-charts.k} | 7 +++-- benchmarks/README.md | 12 ++++---- benchmarks/{simple.k => no-charts.k} | 2 +- 6 files changed, 40 insertions(+), 19 deletions(-) rename benchmarks/{simple-helm-values.k => 10-charts-with-values.k} (96%) rename benchmarks/{simple-helm.k => 10-charts.k} (63%) rename benchmarks/{simple.k => no-charts.k} (97%) diff --git a/README.md b/README.md index def54ca..fe9febc 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ kcl chart update > Approximate values from my Mac Mini M2. -There is a bit of a trade-off. The binary size is larger, and KCL run performance will be worse by an small, absolute amount of time (in my case ~8ms). Meaning, KCL runs with no Helm templates will be slightly slower compared to upstream KCL. +There is a bit of a trade-off. The binary size is larger, and KCL initialization will be slower by an small, absolute amount of time. Meaning, KCL runs with no Helm templates will be slightly slower compared to upstream KCL. See [benchmarks](./benchmarks) for more details. ## Installation diff --git a/Taskfile.yaml b/Taskfile.yaml index 5563824..32a1639 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -135,9 +135,9 @@ tasks: - task: pull-upstream vars: KCLX_BIN: dist/kclipper_{{OS}}_{{ARCH}}*/kcl - KCL_CODE: benchmarks/simple.k - KCL_HELM: benchmarks/simple-helm.k - KCL_HELM_VALUES: benchmarks/simple-helm-values.k + KCL_CODE: benchmarks/no-charts.k + KCL_HELM: benchmarks/10-charts.k + KCL_HELM_VALUES: benchmarks/10-charts-with-values.k cmds: - >- hyperfine --warmup=50 --min-runs=1000 @@ -154,8 +154,8 @@ tasks: desc: Compares benchmark results vars: DOCKER_MOUNTS: "-v ./.tmp:/tmp -v ./benchmarks:/opt/benchmarks" - KCL_CODE: benchmarks/simple.k - KCL_HELM: benchmarks/simple-helm.k + KCL_CODE: benchmarks/no-charts.k + KCL_HELM: benchmarks/10-charts.k cmds: - >- docker run --name=kcl -d --entrypoint=/usr/bin/bash {{.DOCKER_MOUNTS}} @@ -210,3 +210,22 @@ tasks: cmds: - go install golang.org/x/tools/cmd/deadcode@latest - deadcode {{.FLAGS}} {{.PKG}} + + go-bench-init: + desc: Run Go benchmarks with init tracing + cmds: + - | + echo "a=1" | GODEBUG=inittrace=1 kcl run - 2>&1 | awk ' + BEGIN { + printf "%-100s %-15s %-15s %-10s %-10s\n", "Module", "Init Time (ms)", "Clock Time (ms)", "Bytes", "Allocs"; + printf "%-100s %-15s %-15s %-10s %-10s\n", "------", "--------------", "---------------", "-----", "------"; + } + $1 == "init" { + # Extract the relevant fields and format them into a table + module = $2; + init_time = substr($3, 2); # Remove the "@" prefix + clock_time = $5; + bytes = $8; + allocs = $10; + printf "%-100s %-15s %-15s %-10s %-10s\n", module, init_time, clock_time, bytes, allocs; + }' | tail -n +3 | sort -k3,3n diff --git a/benchmarks/simple-helm-values.k b/benchmarks/10-charts-with-values.k similarity index 96% rename from benchmarks/simple-helm-values.k rename to benchmarks/10-charts-with-values.k index 17879fd..01197e2 100644 --- a/benchmarks/simple-helm-values.k +++ b/benchmarks/10-charts-with-values.k @@ -165,11 +165,12 @@ schema PodInfoValues: } } -_chart = helm.template( +_chart = [helm.template( chart="podinfo", + releaseName="podinfo-${x}", target_revision="6.7.1", repo_url="https://stefanprodan.github.io/podinfo", values=PodInfoValues{} -) +) for x in range(0, 10)] -manifests.yaml_stream(_chart) +manifests.yaml_stream([*_chart]) diff --git a/benchmarks/simple-helm.k b/benchmarks/10-charts.k similarity index 63% rename from benchmarks/simple-helm.k rename to benchmarks/10-charts.k index d318c58..7bd84d7 100644 --- a/benchmarks/simple-helm.k +++ b/benchmarks/10-charts.k @@ -1,13 +1,14 @@ import kcl_plugin.helm import manifests -_chart = helm.template( +_chart = [helm.template( chart="podinfo", + releaseName="podinfo-${x}", target_revision="6.7.1", repo_url="https://stefanprodan.github.io/podinfo", values={ ingress.enabled = True } -) +) for x in range(0, 10)] -manifests.yaml_stream(_chart) +manifests.yaml_stream([*_chart]) diff --git a/benchmarks/README.md b/benchmarks/README.md index dcc4bb6..6d01a07 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,6 +1,6 @@ -| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | -| :------------------------------------------- | ---------: | -------: | -------: | ----------: | -| `.tmp/bin/kcl ./benchmarks/simple.k` | 24.6 ± 3.0 | 22.5 | 51.9 | 1.00 | -| `kclipper ./benchmarks/simple.k` | 36.8 ± 0.7 | 34.7 | 38.5 | 1.50 ± 0.18 | -| `kclipper ./benchmarks/simple-helm.k` | 40.8 ± 3.1 | 37.7 | 63.9 | 1.66 ± 0.24 | -| `kclipper ./benchmarks/simple-helm-values.k` | 42.7 ± 0.9 | 40.2 | 45.0 | 1.73 ± 0.21 | +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| :---------------------------------------------- | ---------: | -------: | -------: | ----------: | +| `.tmp/bin/kcl ./benchmarks/no-charts.k` | 46.0 ± 2.4 | 43.5 | 82.7 | 1.00 | +| `kclipper ./benchmarks/no-charts.k` | 58.5 ± 2.6 | 56.2 | 94.5 | 1.27 ± 0.09 | +| `kclipper ./benchmarks/10-charts.k` | 80.4 ± 2.9 | 77.3 | 116.3 | 1.75 ± 0.11 | +| `kclipper ./benchmarks/10-charts-with-values.k` | 87.8 ± 3.5 | 84.0 | 126.6 | 1.91 ± 0.12 | diff --git a/benchmarks/simple.k b/benchmarks/no-charts.k similarity index 97% rename from benchmarks/simple.k rename to benchmarks/no-charts.k index 40ee3a5..38c4eb4 100644 --- a/benchmarks/simple.k +++ b/benchmarks/no-charts.k @@ -35,6 +35,6 @@ _notAChart = [{ serviceAccountName = "default" } } -} for x in range(0, 5)] +} for x in range(0, 50)] manifests.yaml_stream(_notAChart, {})