From 64595a91998d8d06276163ab2a461142d4fc6ae2 Mon Sep 17 00:00:00 2001
From: Menno Finlay-Smits <inbox@menno.io>
Date: Tue, 3 Apr 2018 15:33:30 +1200
Subject: [PATCH 1/3] perfcheck: Try multiple (shorter) attempts

To mitigate against the variation in performance in TravisCI's test
environment, perfcheck will now try multiple attempts at passing. Each
attempt is 3 iterations of benchmark runs.

This increases the chances of success and makes the overall test run
time shorter if the first attempt passes.
---
 perfcheck | 42 ++++++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/perfcheck b/perfcheck
index dc49cff..8235ff6 100755
--- a/perfcheck
+++ b/perfcheck
@@ -3,7 +3,8 @@
 GO_PACKAGE=github.com/jumptrading/influx-spout
 REFERENCE_REVISION=${REFERENCE_REVISION:-aa46d4677bb8ef72848c211312d9051b879322dc}
 
-iterations=5
+max_attempts=3
+iterations=3
 orig_gopath=$GOPATH
 
 echo "Comparing working tree against $REFERENCE_REVISION"
@@ -70,25 +71,30 @@ git checkout --quiet -b perfcheck $REFERENCE_REVISION > /dev/null
 popd > /dev/null
 set +e
 
-# Remove output from previous runs
-rm -f $reference_bench_output $current_bench_output
 
 # Run the tests for the current benchmarks and reference benchmarks
 # $iterations times. The runs are interleaved to minimise the effects
 # of other load on the host.
-for i in `seq $iterations`; do
-    title "Iteration $i/$iterations"
-
-    title "Running current benchmarks"
-    export GOPATH=$orig_gopath
-    capture_benchmarks "$test_sizes" $current_bench_output
-
-    title "Running reference benchmarks"
-    export GOPATH=$ref_gopath
-    pushd $clone_dir > /dev/null
-    capture_benchmarks "$test_sizes" $reference_bench_output
-    popd > /dev/null
-done
+for a in `seq $max_attempts`; do
+    title "Attempt $a ($max_attempts max)"
+
+    # Remove output from previous runs
+    rm -f $reference_bench_output $current_bench_output
+
+    for i in `seq $iterations`; do
+        title "Iteration $i/$iterations"
 
-title "Comparing benchmarks"
-./benchcheck/benchcheck $reference_bench_output $current_bench_output
+        title "Running current benchmarks"
+        export GOPATH=$orig_gopath
+        capture_benchmarks "$test_sizes" $current_bench_output
+
+        title "Running reference benchmarks"
+        export GOPATH=$ref_gopath
+        pushd $clone_dir > /dev/null
+        capture_benchmarks "$test_sizes" $reference_bench_output
+        popd > /dev/null
+    done
+
+    title "Comparing benchmarks"
+    ./benchcheck/benchcheck $reference_bench_output $current_bench_output && exit 0
+done

From bd8cee03df56ca9f1fde11719e3f1f72c36db8dc Mon Sep 17 00:00:00 2001
From: Menno Finlay-Smits <inbox@menno.io>
Date: Tue, 3 Apr 2018 16:10:52 +1200
Subject: [PATCH 2/3] perfcheck: Have benchcheck always print results

While we're working through perfcheck changes it's useful to always
see the benchmark comparison from benchcheck.
---
 perfcheck | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perfcheck b/perfcheck
index 8235ff6..654839a 100755
--- a/perfcheck
+++ b/perfcheck
@@ -96,5 +96,5 @@ for a in `seq $max_attempts`; do
     done
 
     title "Comparing benchmarks"
-    ./benchcheck/benchcheck $reference_bench_output $current_bench_output && exit 0
+    ./benchcheck/benchcheck -print $reference_bench_output $current_bench_output && exit 0
 done

From 340aebe1397f040d13b160b0310367a1f8374e5b Mon Sep 17 00:00:00 2001
From: Menno Finlay-Smits <inbox@menno.io>
Date: Tue, 3 Apr 2018 16:11:25 +1200
Subject: [PATCH 3/3] benchcheck: Tweak output

Print output table first and then finish with the summary
sentence. The summary sentence is now printed when -print is
passed (previously it wasn't).
---
 benchcheck/benchcheck.go | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/benchcheck/benchcheck.go b/benchcheck/benchcheck.go
index 7379a41..f3aef93 100644
--- a/benchcheck/benchcheck.go
+++ b/benchcheck/benchcheck.go
@@ -67,18 +67,16 @@ func main() {
 		log.Fatal(err)
 	}
 
-	if hasProblem {
-		fmt.Println("Performance regression found!")
-		fmt.Println()
-		printTables(tables)
-	} else if *flagPrint {
+	if hasProblem || *flagPrint {
 		printTables(tables)
-	} else {
-		fmt.Println("No performance regressions found")
 	}
+	fmt.Println()
 
 	if hasProblem {
+		fmt.Println("Performance regression found!")
 		os.Exit(1)
+	} else {
+		fmt.Println("No performance regressions found.")
 	}
 }