create dummy thresholds from ci results

swift-server · Feb 27, 2024 · 34e7b4d · 34e7b4d
1 parent 6a2a1b7
commit 34e7b4d
Show file tree

Hide file tree

Showing 23 changed files with 118 additions and 114 deletions.
diff --git a/Benchmarks/Benchmarks/SwiftKafkaConsumerBenchmarks/KafkaConsumerBenchmark.swift b/Benchmarks/Benchmarks/SwiftKafkaConsumerBenchmarks/KafkaConsumerBenchmark.swift
@@ -34,7 +34,7 @@ let benchmarks = {
         thresholds: [
             // Thresholds are wild guess mostly. Have to adjust with time.
             .wallClock: .init(relative: [.p90: 10]),
-            .cpuTotal: .init(relative: [.p90: 0]),
+            .cpuTotal: .init(relative: [.p90: 10]),
             .allocatedResidentMemory: .init(relative: [.p90: 20]),
             .contextSwitches: .init(relative: [.p90: 10]),
             .throughput: .init(relative: [.p90: 10]),

diff --git a/Benchmarks/Benchmarks/SwiftKafkaProducerBenchmarks/KafkaProducerBenchmark.swift b/Benchmarks/Benchmarks/SwiftKafkaProducerBenchmarks/KafkaProducerBenchmark.swift
@@ -22,7 +22,19 @@ let benchmarks = {
         warmupIterations: 0,
         scalingFactor: .one,
         maxDuration: .seconds(5),
-        maxIterations: 100
+        maxIterations: 100,
+        thresholds: [
+            // Thresholds are wild guess mostly. Have to adjust with time.
+            .wallClock: .init(relative: [.p90: 10]),
+            .cpuTotal: .init(relative: [.p90: 10]),
+            .allocatedResidentMemory: .init(relative: [.p90: 20]),
+            .contextSwitches: .init(relative: [.p90: 10]),
+            .throughput: .init(relative: [.p90: 10]),
+            .objectAllocCount: .init(relative: [.p90: 10]),
+            .retainCount: .init(relative: [.p90: 10]),
+            .releaseCount: .init(relative: [.p90: 10]),
+            .retainReleaseDelta: .init(relative: [.p90: 10]),
+        ]
     )
 
     Benchmark.setup = {}

diff --git a/Benchmarks/Plugins/BenchmarkCheckWrapper/BenchmarkCheckWrapper.swift b/Benchmarks/Plugins/BenchmarkCheckWrapper/BenchmarkCheckWrapper.swift
diff --git a/....10/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json b/....10/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 77660159,
-  "cpuTotal" : 80000000,
-  "objectAllocCount" : 4643,
-  "releaseCount" : 30815,
-  "retainCount" : 23791,
-  "retainReleaseDelta" : 2391,
+  "allocatedResidentMemory" : 77463551,
+  "cpuTotal" : 70057983,
+  "objectAllocCount" : 4647,
+  "releaseCount" : 25903,
+  "retainCount" : 18863,
+  "retainReleaseDelta" : 2399,
   "throughput" : 8,
-  "wallClock" : 133693439
-}
+  "wallClock" : 9994972543
+}
diff --git a/...SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json b/...SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 70254592,
-  "cpuTotal" : 270000000,
-  "objectAllocCount" : 7917,
-  "releaseCount" : 38881,
-  "retainCount" : 27059,
-  "retainReleaseDelta" : 3905,
+  "allocatedResidentMemory" : 72351744,
+  "cpuTotal" : 240000000,
+  "objectAllocCount" : 7980,
+  "releaseCount" : 34126,
+  "retainCount" : 22195,
+  "retainReleaseDelta" : 3951,
   "throughput" : 2,
-  "wallClock" : 634566500
-}
+  "wallClock" : 9995008416
+}
diff --git a/...sholds/5.10/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json b/...sholds/5.10/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 42860544,
+  "allocatedResidentMemory" : 46530560,
   "cpuTotal" : 20000000,
   "objectAllocCount" : 0,
   "releaseCount" : 0,
   "retainCount" : 0,
   "retainReleaseDelta" : 0,
   "throughput" : 2,
-  "wallClock" : 620173292
-}
+  "wallClock" : 621817625
+}
diff --git a/...ds/5.10/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json b/...ds/5.10/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 48300032,
-  "cpuTotal" : 150000000,
+  "allocatedResidentMemory" : 48234496,
+  "cpuTotal" : 400000000,
   "objectAllocCount" : 0,
   "releaseCount" : 0,
   "retainCount" : 0,
   "retainReleaseDelta" : 0,
   "throughput" : 1,
-  "wallClock" : 827764833
-}
+  "wallClock" : 1506038376
+}
diff --git a/...5.7/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json b/...5.7/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 69468160,
-  "cpuTotal" : 270000000,
-  "objectAllocCount" : 7175,
-  "releaseCount" : 37657,
-  "retainCount" : 26402,
-  "retainReleaseDelta" : 4080,
-  "throughput" : 2,
-  "wallClock" : 683494251
-}
+  "allocatedResidentMemory" : 77463551,
+  "cpuTotal" : 70057983,
+  "objectAllocCount" : 4647,
+  "releaseCount" : 25903,
+  "retainCount" : 18863,
+  "retainReleaseDelta" : 2399,
+  "throughput" : 8,
+  "wallClock" : 9994972543
+}
diff --git a/...SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json b/...SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 73007104,
-  "cpuTotal" : 270000000,
-  "objectAllocCount" : 7941,
-  "releaseCount" : 39038,
-  "retainCount" : 27174,
-  "retainReleaseDelta" : 3923,
+  "allocatedResidentMemory" : 72351744,
+  "cpuTotal" : 240000000,
+  "objectAllocCount" : 7980,
+  "releaseCount" : 34126,
+  "retainCount" : 22195,
+  "retainReleaseDelta" : 3951,
   "throughput" : 2,
-  "wallClock" : 644024709
-}
+  "wallClock" : 9995008416
+}
diff --git a/...esholds/5.7/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json b/...esholds/5.7/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 46596096,
+  "allocatedResidentMemory" : 46530560,
   "cpuTotal" : 20000000,
   "objectAllocCount" : 0,
   "releaseCount" : 0,
   "retainCount" : 0,
   "retainReleaseDelta" : 0,
   "throughput" : 2,
-  "wallClock" : 627353501
-}
+  "wallClock" : 621817625
+}
diff --git a/...lds/5.7/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json b/...lds/5.7/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 44040192,
-  "cpuTotal" : 150000000,
+  "allocatedResidentMemory" : 48234496,
+  "cpuTotal" : 400000000,
   "objectAllocCount" : 0,
   "releaseCount" : 0,
   "retainCount" : 0,
   "retainReleaseDelta" : 0,
   "throughput" : 1,
-  "wallClock" : 835152709
-}
+  "wallClock" : 1506038376
+}
diff --git a/...5.8/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json b/...5.8/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json
@@ -6,5 +6,5 @@
   "retainCount" : 18863,
   "retainReleaseDelta" : 2399,
   "throughput" : 8,
-  "wallClock" : 132972543
-}
+  "wallClock" : 9994972543
+}
diff --git a/...SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json b/...SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json
@@ -6,5 +6,5 @@
   "retainCount" : 22195,
   "retainReleaseDelta" : 3951,
   "throughput" : 2,
-  "wallClock" : 629008416
-}
+  "wallClock" : 9995008416
+}
diff --git a/...esholds/5.8/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json b/...esholds/5.8/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
   "allocatedResidentMemory" : 46530560,
-  "cpuTotal" : 10000000,
+  "cpuTotal" : 20000000,
   "objectAllocCount" : 0,
   "releaseCount" : 0,
   "retainCount" : 0,
   "retainReleaseDelta" : 0,
   "throughput" : 2,
   "wallClock" : 621817625
-}
+}
diff --git a/...lds/5.8/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json b/...lds/5.8/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
   "allocatedResidentMemory" : 48234496,
-  "cpuTotal" : 150000000,
+  "cpuTotal" : 400000000,
   "objectAllocCount" : 0,
   "releaseCount" : 0,
   "retainCount" : 0,
   "retainReleaseDelta" : 0,
   "throughput" : 1,
-  "wallClock" : 826038376
-}
+  "wallClock" : 1506038376
+}
diff --git a/...5.9/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json b/...5.9/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 72876032,
-  "cpuTotal" : 270000000,
-  "objectAllocCount" : 6921,
-  "releaseCount" : 31952,
-  "retainCount" : 21121,
-  "retainReleaseDelta" : 3910,
-  "throughput" : 2,
-  "wallClock" : 639700000
-}
+  "allocatedResidentMemory" : 77463551,
+  "cpuTotal" : 70057983,
+  "objectAllocCount" : 4647,
+  "releaseCount" : 25903,
+  "retainCount" : 18863,
+  "retainReleaseDelta" : 2399,
+  "throughput" : 8,
+  "wallClock" : 9994972543
+}
diff --git a/...SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json b/...SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 72089600,
-  "cpuTotal" : 260000000,
-  "objectAllocCount" : 7907,
-  "releaseCount" : 33901,
-  "retainCount" : 22094,
-  "retainReleaseDelta" : 3900,
+  "allocatedResidentMemory" : 72351744,
+  "cpuTotal" : 240000000,
+  "objectAllocCount" : 7980,
+  "releaseCount" : 34126,
+  "retainCount" : 22195,
+  "retainReleaseDelta" : 3951,
   "throughput" : 2,
-  "wallClock" : 631249876
-}
+  "wallClock" : 9995008416
+}
diff --git a/...esholds/5.9/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json b/...esholds/5.9/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 43319296,
+  "allocatedResidentMemory" : 46530560,
   "cpuTotal" : 20000000,
   "objectAllocCount" : 0,
   "releaseCount" : 0,
   "retainCount" : 0,
   "retainReleaseDelta" : 0,
   "throughput" : 2,
-  "wallClock" : 620977209
-}
+  "wallClock" : 621817625
+}
diff --git a/...lds/5.9/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json b/...lds/5.9/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 44761088,
-  "cpuTotal" : 150000000,
+  "allocatedResidentMemory" : 48234496,
+  "cpuTotal" : 400000000,
   "objectAllocCount" : 0,
   "releaseCount" : 0,
   "retainCount" : 0,
   "retainReleaseDelta" : 0,
   "throughput" : 1,
-  "wallClock" : 819931167
-}
+  "wallClock" : 1506038376
+}
diff --git a/...ain/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json b/...ain/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 75759616,
-  "cpuTotal" : 280000000,
-  "objectAllocCount" : 6919,
-  "releaseCount" : 36876,
-  "retainCount" : 26056,
-  "retainReleaseDelta" : 3909,
-  "throughput" : 2,
-  "wallClock" : 636520708
-}
+  "allocatedResidentMemory" : 77463551,
+  "cpuTotal" : 70057983,
+  "objectAllocCount" : 4647,
+  "releaseCount" : 25903,
+  "retainCount" : 18863,
+  "retainReleaseDelta" : 2399,
+  "throughput" : 8,
+  "wallClock" : 9994972543
+}
diff --git a/...SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json b/...SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 72744960,
-  "cpuTotal" : 290000000,
-  "objectAllocCount" : 7906,
-  "releaseCount" : 38839,
-  "retainCount" : 27022,
-  "retainReleaseDelta" : 3921,
+  "allocatedResidentMemory" : 72351744,
+  "cpuTotal" : 240000000,
+  "objectAllocCount" : 7980,
+  "releaseCount" : 34126,
+  "retainCount" : 22195,
+  "retainReleaseDelta" : 3951,
   "throughput" : 2,
-  "wallClock" : 634470084
-}
+  "wallClock" : 9995008416
+}
diff --git a/...sholds/main/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json b/...sholds/main/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 50200576,
+  "allocatedResidentMemory" : 46530560,
   "cpuTotal" : 20000000,
   "objectAllocCount" : 0,
   "releaseCount" : 0,
   "retainCount" : 0,
   "retainReleaseDelta" : 0,
   "throughput" : 2,
-  "wallClock" : 618693125
-}
+  "wallClock" : 621817625
+}
diff --git a/...ds/main/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json b/...ds/main/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json
@@ -1,10 +1,10 @@
 {
-  "allocatedResidentMemory" : 47185920,
-  "cpuTotal" : 150000000,
+  "allocatedResidentMemory" : 48234496,
+  "cpuTotal" : 400000000,
   "objectAllocCount" : 0,
   "releaseCount" : 0,
   "retainCount" : 0,
   "retainReleaseDelta" : 0,
   "throughput" : 1,
-  "wallClock" : 815092876
-}
+  "wallClock" : 1506038376
+}