Update benchmarks

MacroPower · Jan 4, 2025 · 7a44b55 · 7a44b55
1 parent 57b5098
commit 7a44b55
Show file tree

Hide file tree

Showing 6 changed files with 40 additions and 19 deletions.
diff --git a/README.md b/README.md
@@ -84,7 +84,7 @@ kcl chart update
 
 > Approximate values from my Mac Mini M2.
 
-There is a bit of a trade-off. The binary size is larger, and KCL run performance will be worse by an small, absolute amount of time (in my case ~8ms). Meaning, KCL runs with no Helm templates will be slightly slower compared to upstream KCL.
+There is a bit of a trade-off. The binary size is larger, and KCL initialization will be slower by an small, absolute amount of time. Meaning, KCL runs with no Helm templates will be slightly slower compared to upstream KCL. See [benchmarks](./benchmarks) for more details.
 
 ## Installation
 

diff --git a/Taskfile.yaml b/Taskfile.yaml
@@ -135,9 +135,9 @@ tasks:
       - task: pull-upstream
     vars:
       KCLX_BIN: dist/kclipper_{{OS}}_{{ARCH}}*/kcl
-      KCL_CODE: benchmarks/simple.k
-      KCL_HELM: benchmarks/simple-helm.k
-      KCL_HELM_VALUES: benchmarks/simple-helm-values.k
+      KCL_CODE: benchmarks/no-charts.k
+      KCL_HELM: benchmarks/10-charts.k
+      KCL_HELM_VALUES: benchmarks/10-charts-with-values.k
     cmds:
       - >-
         hyperfine --warmup=50 --min-runs=1000
@@ -154,8 +154,8 @@ tasks:
     desc: Compares benchmark results
     vars:
       DOCKER_MOUNTS: "-v ./.tmp:/tmp -v ./benchmarks:/opt/benchmarks"
-      KCL_CODE: benchmarks/simple.k
-      KCL_HELM: benchmarks/simple-helm.k
+      KCL_CODE: benchmarks/no-charts.k
+      KCL_HELM: benchmarks/10-charts.k
     cmds:
       - >-
         docker run --name=kcl -d --entrypoint=/usr/bin/bash {{.DOCKER_MOUNTS}}
@@ -210,3 +210,22 @@ tasks:
     cmds:
       - go install golang.org/x/tools/cmd/deadcode@latest
       - deadcode {{.FLAGS}} {{.PKG}}
+
+  go-bench-init:
+    desc: Run Go benchmarks with init tracing
+    cmds:
+      - |
+        echo "a=1" | GODEBUG=inittrace=1 kcl run - 2>&1 | awk '
+        BEGIN {
+            printf "%-100s %-15s %-15s %-10s %-10s\n", "Module", "Init Time (ms)", "Clock Time (ms)", "Bytes", "Allocs";
+            printf "%-100s %-15s %-15s %-10s %-10s\n", "------", "--------------", "---------------", "-----", "------";
+        }
+        $1 == "init" {
+            # Extract the relevant fields and format them into a table
+            module = $2;
+            init_time = substr($3, 2);  # Remove the "@" prefix
+            clock_time = $5;
+            bytes = $8;
+            allocs = $10;
+            printf "%-100s %-15s %-15s %-10s %-10s\n", module, init_time, clock_time, bytes, allocs;
+        }' | tail -n +3 | sort -k3,3n
diff --git a/benchmarks/simple-helm-values.k → benchmarks/10-charts-with-values.k b/benchmarks/simple-helm-values.k → benchmarks/10-charts-with-values.k
@@ -165,11 +165,12 @@ schema PodInfoValues:
         }
     }
 
-_chart = helm.template(
+_chart = [helm.template(
     chart="podinfo",
+    releaseName="podinfo-${x}",
     target_revision="6.7.1",
     repo_url="https://stefanprodan.github.io/podinfo",
     values=PodInfoValues{}
-)
+) for x in range(0, 10)]
 
-manifests.yaml_stream(_chart)
+manifests.yaml_stream([*_chart])
diff --git a/benchmarks/simple-helm.k → benchmarks/10-charts.k b/benchmarks/simple-helm.k → benchmarks/10-charts.k
@@ -1,13 +1,14 @@
 import kcl_plugin.helm
 import manifests
 
-_chart = helm.template(
+_chart = [helm.template(
     chart="podinfo",
+    releaseName="podinfo-${x}",
     target_revision="6.7.1",
     repo_url="https://stefanprodan.github.io/podinfo",
     values={
         ingress.enabled = True
     }
-)
+) for x in range(0, 10)]
 
-manifests.yaml_stream(_chart)
+manifests.yaml_stream([*_chart])
diff --git a/benchmarks/README.md b/benchmarks/README.md
@@ -1,6 +1,6 @@
-| Command                                      |  Mean [ms] | Min [ms] | Max [ms] |    Relative |
-| :------------------------------------------- | ---------: | -------: | -------: | ----------: |
-| `.tmp/bin/kcl ./benchmarks/simple.k`         | 24.6 ± 3.0 |     22.5 |     51.9 |        1.00 |
-| `kclipper ./benchmarks/simple.k`             | 36.8 ± 0.7 |     34.7 |     38.5 | 1.50 ± 0.18 |
-| `kclipper ./benchmarks/simple-helm.k`        | 40.8 ± 3.1 |     37.7 |     63.9 | 1.66 ± 0.24 |
-| `kclipper ./benchmarks/simple-helm-values.k` | 42.7 ± 0.9 |     40.2 |     45.0 | 1.73 ± 0.21 |
+| Command                                         |  Mean [ms] | Min [ms] | Max [ms] |    Relative |
+| :---------------------------------------------- | ---------: | -------: | -------: | ----------: |
+| `.tmp/bin/kcl ./benchmarks/no-charts.k`         | 46.0 ± 2.4 |     43.5 |     82.7 |        1.00 |
+| `kclipper ./benchmarks/no-charts.k`             | 58.5 ± 2.6 |     56.2 |     94.5 | 1.27 ± 0.09 |
+| `kclipper ./benchmarks/10-charts.k`             | 80.4 ± 2.9 |     77.3 |    116.3 | 1.75 ± 0.11 |
+| `kclipper ./benchmarks/10-charts-with-values.k` | 87.8 ± 3.5 |     84.0 |    126.6 | 1.91 ± 0.12 |
diff --git a/benchmarks/simple.k → benchmarks/no-charts.k b/benchmarks/simple.k → benchmarks/no-charts.k
@@ -35,6 +35,6 @@ _notAChart = [{
             serviceAccountName = "default"
         }
     }
-} for x in range(0, 5)]
+} for x in range(0, 50)]
 
 manifests.yaml_stream(_notAChart, {})