Skip to content
This repository has been archived by the owner on Mar 9, 2022. It is now read-only.

Commit

Permalink
Merge pull request #38 from mjs/enable-perfcheck
Browse files Browse the repository at this point in the history
Run performance regression checks in TravisCI
  • Loading branch information
oplehto authored Feb 27, 2018
2 parents 708e3d0 + 8257202 commit 5249beb
Show file tree
Hide file tree
Showing 785 changed files with 12,609 additions and 184,553 deletions.
4 changes: 3 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,6 @@ go:
# No need for an install step as all dependencies are vendored.
install: []

script: ./runtests -r small medium large
script:
- ./runtests -r small medium large
- ./perfcheck small medium large
68 changes: 54 additions & 14 deletions Gopkg.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions Gopkg.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,7 @@
[[constraint]]
name = "github.com/stretchr/testify"
version = "1.1.4"

[[constraint]]
branch = "master"
name = "golang.org/x/perf"
217 changes: 60 additions & 157 deletions benchcheck/benchcheck.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2017 Jump Trading
// Copyright 2018 Jump Trading
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -11,194 +11,97 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// This file contains source code adapted from software developed for
// the Go project (see https://github.com/golang/tools/).

package main

import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"sort"
"strconv"
"text/tabwriter"

"golang.org/x/tools/benchmark/parse"
)
"strings"

var (
theshPercent = flag.Int("threshold", 10, "maximum allowed regression (percent)")
"golang.org/x/perf/benchstat"
)

const usageFooter = `
Each input file should be from:
go test -run=NONE -bench=. > [old,new].txt
Benchmark compares old and new for each benchmark and fails if the new
benchmarks are some percentage theshold slower.
`

func threshExceeded(d Delta) bool {
p := 100*d.Float64() - 100
return p > float64(*theshPercent)
func usage() {
fmt.Fprintf(os.Stderr, "usage: benchcheck [options] old.txt new.txt\n")
fmt.Fprintf(os.Stderr, "options:\n")
flag.PrintDefaults()
os.Exit(2)
}

func main() {
os.Exit(runMain())
}
var flagAlpha = flag.Float64("alpha", 0.05, "consider change significant if p < `α`")
var flagDelta = flag.Float64("max-delta", 10, "error if delta % larger than this")
var flagPrint = flag.Bool("print", false, "print output even when no regression found")

func runMain() int {
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "usage: %s old.txt new.txt\n\n", os.Args[0])
flag.PrintDefaults()
fmt.Fprint(os.Stderr, usageFooter)
os.Exit(2)
}
func main() {
log.SetPrefix("benchstat: ")
log.SetFlags(0)
flag.Usage = usage
flag.Parse()
if flag.NArg() != 2 {
flag.Usage()
}

before := parseFile(flag.Arg(0))
after := parseFile(flag.Arg(1))

cmps, warnings := Correlate(before, after)
for _, warn := range warnings {
fmt.Fprintln(os.Stderr, warn)
}
if len(cmps) == 0 {
fatal("benchcmp: no repeated benchmarks")
c := &benchstat.Collection{
Alpha: *flagAlpha,
DeltaTest: benchstat.UTest,
}
sort.Slice(cmps, func(i, j int) bool {
return cmps[i].Before.Ord < cmps[j].Before.Ord
})

w := new(tabwriter.Writer)
w.Init(os.Stdout, 0, 0, 5, ' ', 0)
defer w.Flush()

var regressions int
var header bool // Has the header has been displayed yet for a given block?

for _, cmp := range cmps {
if !cmp.Measured(parse.NsPerOp) {
continue
}
delta := cmp.DeltaNsPerOp()
if threshExceeded(delta) {
regressions++
if !header {
fmt.Fprint(w, "benchmark\told ns/op\tnew ns/op\tdelta\n")
header = true
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n",
cmp.Name(),
formatNs(cmp.Before.NsPerOp),
formatNs(cmp.After.NsPerOp),
delta.Percent(),
)
for _, file := range flag.Args() {
data, err := ioutil.ReadFile(file)
if err != nil {
log.Fatal(err)
}
c.AddConfig(file, data)
}

for _, cmp := range cmps {
if !cmp.Measured(parse.MBPerS) {
continue
}
delta := cmp.DeltaMBPerS()
if threshExceeded(delta) {
regressions++
if !header {
fmt.Fprint(w, "\nbenchmark\told MB/s\tnew MB/s\tspeedup\n")
header = true
}
fmt.Fprintf(w, "%s\t%.2f\t%.2f\t%s\n", cmp.Name(), cmp.Before.MBPerS, cmp.After.MBPerS, delta.Multiple())
}
}
tables := c.Tables()

header = false
for _, cmp := range cmps {
if !cmp.Measured(parse.AllocsPerOp) {
continue
}
delta := cmp.DeltaAllocsPerOp()
if threshExceeded(delta) {
regressions++
if !header {
fmt.Fprint(w, "\nbenchmark\told allocs\tnew allocs\tdelta\n")
header = true
}
fmt.Fprintf(w, "%s\t%d\t%d\t%s\n",
cmp.Name(),
cmp.Before.AllocsPerOp,
cmp.After.AllocsPerOp,
delta.Percent(),
)
}
}

header = false
for _, cmp := range cmps {
if !cmp.Measured(parse.AllocedBytesPerOp) {
continue
}
delta := cmp.DeltaAllocedBytesPerOp()
if threshExceeded(delta) {
regressions++
if !header {
fmt.Fprint(w, "\nbenchmark\told bytes\tnew bytes\tdelta\n")
header = true
}
fmt.Fprintf(w, "%s\t%d\t%d\t%s\n",
cmp.Name(),
cmp.Before.AllocedBytesPerOp,
cmp.After.AllocedBytesPerOp,
delta.Percent(),
)
}
// Look for problematic benchmarks
hasProblem, err := findProblems(tables)
if err != nil {
log.Fatal(err)
}

w.Flush()

if regressions > 0 {
if hasProblem {
fmt.Println("Performance regression found!")
fmt.Println()
printTables(tables)
} else if *flagPrint {
printTables(tables)
} else {
fmt.Println("No performance regressions found")
}
fmt.Printf("%d benchmarks compared.\n", len(cmps))
if regressions > 0 {
fmt.Fprintf(os.Stderr, "%d performance regressions detected.\n", regressions)
return 1
}
return 0
}

func fatal(msg interface{}) {
fmt.Fprintln(os.Stderr, msg)
os.Exit(1)
if hasProblem {
os.Exit(1)
}
}

func parseFile(path string) parse.Set {
f, err := os.Open(path)
if err != nil {
fatal(err)
}
defer f.Close()
bb, err := parse.ParseSet(f)
if err != nil {
fatal(err)
func findProblems(tables []*benchstat.Table) (bool, error) {
for _, table := range tables {
for _, row := range table.Rows {
if row.Delta == "~" {
continue
}
delta, err := strconv.ParseFloat(strings.TrimRight(row.Delta, "%"), 64)
if err != nil {
return false, err
}
if delta > *flagDelta {
return true, nil
}
}
}
return bb
return false, nil
}

// formatNs formats ns measurements to expose a useful amount of
// precision. It mirrors the ns precision logic of testing.B.
func formatNs(ns float64) string {
prec := 0
switch {
case ns < 10:
prec = 2
case ns < 100:
prec = 1
}
return strconv.FormatFloat(ns, 'f', prec, 64)
func printTables(tables []*benchstat.Table) {
var buf bytes.Buffer
benchstat.FormatText(&buf, tables)
os.Stdout.Write(buf.Bytes())
}
Loading

0 comments on commit 5249beb

Please sign in to comment.