test: refactor to benchmark suite and add test-benchmark target (#383)

#345

*Description of changes:*
Refactor to benchmark suite so we can insert vm init between vm
benchmark and container benchmark.

*Testing done:*
```
✗ make test-benchmark-container
cd benchmark/container && go test -ldflags "-X github.com/runfinch/finch/pkg/version.Version=v0.6.0-2-g4b15328.modified -X github.com/runfinch/finch/pkg/version.GitCommit=4b15328755164eeea3549dbc02926e0fd4e1e14f.m" -bench=. -benchmem --installed="false"
goos: darwin
goarch: arm64
pkg: github.com/runfinch/finch/benchmark/container
BenchmarkContainer/BenchmarkContainerRun-8                     3         429230791 ns/op                 1.822 %cpu_avg/op         41.41 %cpu_peak/op               0.4292 cpu_seconds/op       -2730 disk_bytes/op           27608 B/op           464 allocs/op
BenchmarkContainer/BenchmarkImageBuild-8                       1        1687634542 ns/op                 1.396 %cpu_avg/op         85.71 %cpu_peak/op               1.688 cpu_seconds/op      5341184 disk_bytes/op           83360 B/op          1668 allocs/op
PASS
ok      github.com/runfinch/finch/benchmark/container   12.491s

✗ make test-benchmark-vm
cd benchmark/vm && go test -ldflags "-X github.com/runfinch/finch/pkg/version.Version=v0.6.0-2-g4b15328.modified -X github.com/runfinch/finch/pkg/version.GitCommit=4b15328755164eeea3549dbc02926e0fd4e1e14f.m" -bench=. -benchmem --installed="false"
goos: darwin
goarch: arm64
pkg: github.com/runfinch/finch/benchmark/vm
BenchmarkVM/BenchmarkVMInit-8                  1        68261909667 ns/op                1.473 %cpu_avg/op             100.0 %cpu_peak/op          68.26 cpu_seconds/op    1472135168 disk_bytes/op         3097904 B/op      65977 allocs/op
BenchmarkVM/BenchmarkVMStart-8                 1        23871741458 ns/op                1.143 %cpu_avg/op             100.0 %cpu_peak/op          23.87 cpu_seconds/op       1945600 disk_bytes/op         1079840 B/op      23035 allocs/op
PASS
ok      github.com/runfinch/finch/benchmark/vm  177.623s

✗ make test-benchmark   
cd benchmark/all && go test -ldflags "-X github.com/runfinch/finch/pkg/version.Version=v0.6.0-2-g4b15328.modified -X github.com/runfinch/finch/pkg/version.GitCommit=4b15328755164eeea3549dbc02926e0fd4e1e14f.m" -bench=. -benchmem --installed="false"
goos: darwin
goarch: arm64
pkg: github.com/runfinch/finch/benchmark/all
BenchmarkAll/BenchmarkVMInit-8                 1        70063292500 ns/op                0.9251 %cpu_avg/op            100.0 %cpu_peak/op          70.06 cpu_seconds/op    1439682560 disk_bytes/op         3175968 B/op      67751 allocs/op
BenchmarkAll/BenchmarkVMStart-8                1        23542149250 ns/op                1.204 %cpu_avg/op             100.0 %cpu_peak/op          23.54 cpu_seconds/op        942080 disk_bytes/op         1066296 B/op      22697 allocs/op
BenchmarkAll/BenchmarkContainerRun-8           3         425348166 ns/op                 2.545 %cpu_avg/op              44.17 %cpu_peak/op          0.4253 cpu_seconds/op           0 disk_bytes/op           27061 B/op        459 allocs/op
BenchmarkAll/BenchmarkImageBuild-8             1        3012537750 ns/op                 0.5392 %cpu_avg/op             31.08 %cpu_peak/op          3.013 cpu_seconds/op      3842048 disk_bytes/op          144184 B/op       2941 allocs/op
PASS
ok      github.com/runfinch/finch/benchmark/all 240.898s

```



- [ X ] I've reviewed the guidance in CONTRIBUTING.md


#### License Acceptance

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

---------

Signed-off-by: Ziwen Ning <ningziwe@amazon.com>

Makefile

@@ -268,6 +268,10 @@ test-e2e-container:
 test-e2e-vm:
 	go test -ldflags $(LDFLAGS) -timeout 45m ./e2e/vm -test.v -ginkgo.v --installed="$(INSTALLED)"
 
+.PHONY: test-benchmark
+test-benchmark:
+	cd benchmark/all && go test -ldflags $(LDFLAGS) -bench=. -benchmem --installed="$(INSTALLED)"
+
 .PHONY: test-benchmark-vm
 test-benchmark-vm:
 	cd benchmark/vm && go test -ldflags $(LDFLAGS) -bench=. -benchmem --installed="$(INSTALLED)"

benchmark/all/all_test.go

@@ -0,0 +1,49 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package all runs all the benchmark tests of Finch.
+package all
+
+import (
+	"testing"
+
+	"github.com/runfinch/finch/benchmark"
+)
+
+func BenchmarkAll(b *testing.B) {
+	suite := &benchmark.Suite{}
+	err := suite.Setup()
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.Run("BenchmarkVMInit", func(b *testing.B) {
+		suite.BenchmarkVMInit(b)
+	})
+
+	b.Run("BenchmarkVMStart", func(b *testing.B) {
+		suite.BenchmarkVMStart(b)
+	})
+
+	err = suite.InitVM()
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.Run("BenchmarkContainerRun", func(b *testing.B) {
+		suite.BenchmarkContainerRun(b)
+	})
+
+	b.Run("BenchmarkImageBuild", func(b *testing.B) {
+		suite.BenchmarkImageBuild(b)
+	})
+
+	err = suite.StopVM()
+	if err != nil {
+		b.Fatal(err)
+	}
+	err = suite.RemoveVM()
+	if err != nil {
+		b.Fatal(err)
+	}
+}

benchmark/container/container_test.go

@@ -5,57 +5,23 @@
 package container
 
 import (
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/runfinch/finch/benchmark"
 )
 
-const (
-	virtualMachineRootCmd = "vm"
-	alpineImage           = "public.ecr.aws/docker/library/alpine:latest"
-	testImageName         = "test:tag"
-	testContainerName     = "ctr-test"
-)
-
-func BenchmarkContainerRun(b *testing.B) {
-	subject, err := benchmark.GetSubject()
+func BenchmarkContainer(b *testing.B) {
+	suite := &benchmark.Suite{}
+	err := suite.Setup()
 	if err != nil {
 		b.Fatal(err)
 	}
-	assert.NoError(b, exec.Command(subject, "pull", alpineImage).Run()) //nolint:gosec // testing only
-	benchmark.Wrapper(b, func() {
-		assert.NoError(b, exec.Command(subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
-	}, func() {
-		assert.NoError(b, exec.Command(subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
+
+	b.Run("BenchmarkContainerRun", func(b *testing.B) {
+		suite.BenchmarkContainerRun(b)
 	})
-	assert.NoError(b, exec.Command(subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
-}
 
-func BenchmarkImageBuild(b *testing.B) {
-	subject, err := benchmark.GetSubject()
-	if err != nil {
-		b.Fatal(err)
-	}
-	homeDir, err := os.UserHomeDir()
-	assert.NoError(b, err)
-	tempDir, err := os.MkdirTemp(homeDir, "finch-test")
-	assert.NoError(b, err)
-	dockerFilePath := filepath.Join(tempDir, "Dockerfile")
-	err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s
-			CMD ["echo", "finch-test-dummy-output"]
-			`, alpineImage)), 0o644)
-	assert.NoError(b, err)
-	buildContext := filepath.Dir(dockerFilePath)
-	defer os.RemoveAll(buildContext) //nolint:errcheck // testing only
-	benchmark.Wrapper(b, func() {
-		assert.NoError(b, exec.Command(subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
-	}, func() {
-		assert.NoError(b, exec.Command(subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
+	b.Run("BenchmarkImageBuild", func(b *testing.B) {
+		suite.BenchmarkImageBuild(b)
 	})
 }

benchmark/suite.go

@@ -0,0 +1,111 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package benchmark runs benchmark tests of Finch.
+package benchmark
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	virtualMachineRootCmd = "vm"
+	alpineImage           = "public.ecr.aws/docker/library/alpine:latest"
+	testImageName         = "test:tag"
+	testContainerName     = "ctr-test"
+)
+
+// Suite is a struct that groups benchmark functions and shared state.
+type Suite struct {
+	subject string
+}
+
+// Setup initializes the Suite by getting the subject.
+func (suite *Suite) Setup() error {
+	subject, err := GetSubject()
+	if err != nil {
+		return err
+	}
+	suite.subject = subject
+	return nil
+}
+
+// InitVM initializes a virtual machine.
+func (suite *Suite) InitVM() error {
+	return exec.Command(suite.subject, virtualMachineRootCmd, "init").Run() // #nosec G204
+}
+
+// StartVM starts the VM.
+func (suite *Suite) StartVM() error {
+	return exec.Command(suite.subject, virtualMachineRootCmd, "start", "-f").Run() // #nosec G204
+}
+
+// StopVM stops the VM.
+func (suite *Suite) StopVM() error {
+	return exec.Command(suite.subject, virtualMachineRootCmd, "stop", "-f").Run() // #nosec G204
+}
+
+// RemoveVM removes the VM.
+func (suite *Suite) RemoveVM() error {
+	return exec.Command(suite.subject, virtualMachineRootCmd, "remove", "-f").Run() // #nosec G204
+}
+
+// BenchmarkVMInit measures the metrics to initialize the VM.
+func (suite *Suite) BenchmarkVMInit(b *testing.B) {
+	Wrapper(b, func() {
+		assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "init").Run()) //nolint:gosec // testing only
+	}, func() {
+		assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "stop", "-f").Run())   //nolint:gosec // testing only
+		assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "remove", "-f").Run()) //nolint:gosec // testing only	})
+	})
+}
+
+// BenchmarkVMStart measures the metrics to start the VM.
+func (suite *Suite) BenchmarkVMStart(b *testing.B) {
+	assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "init").Run())       //nolint:gosec // testing only
+	assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only
+	b.ResetTimer()
+	Wrapper(b, func() {
+		assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "start").Run()) //nolint:gosec // testing only
+	}, func() {
+		assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only
+	})
+	assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "remove", "-f").Run()) //nolint:gosec // testing only
+}
+
+// BenchmarkContainerRun measures the metrics to run a container.
+func (suite *Suite) BenchmarkContainerRun(b *testing.B) {
+	assert.NoError(b, exec.Command(suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only
+	Wrapper(b, func() {
+		assert.NoError(b, exec.Command(suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
+	}, func() {
+		assert.NoError(b, exec.Command(suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
+	})
+	assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
+}
+
+// BenchmarkImageBuild measures the metrics to build an image.
+func (suite *Suite) BenchmarkImageBuild(b *testing.B) {
+	homeDir, err := os.UserHomeDir()
+	assert.NoError(b, err)
+	tempDir, err := os.MkdirTemp(homeDir, "finch-test")
+	assert.NoError(b, err)
+	dockerFilePath := filepath.Join(tempDir, "Dockerfile")
+	err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s
+			CMD ["echo", "finch-test-dummy-output"]
+			`, alpineImage)), 0o644)
+	assert.NoError(b, err)
+	buildContext := filepath.Dir(dockerFilePath)
+	defer os.RemoveAll(buildContext) //nolint:errcheck // testing only
+	Wrapper(b, func() {
+		assert.NoError(b, exec.Command(suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
+	}, func() {
+		assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
+	})
+}

benchmark/vm/vm_test.go

@@ -5,43 +5,23 @@
 package vm
 
 import (
-	"os/exec"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/runfinch/finch/benchmark"
 )
 
-const (
-	virtualMachineRootCmd = "vm"
-)
-
-func BenchmarkVMInit(b *testing.B) {
-	subject, err := benchmark.GetSubject()
+func BenchmarkVM(b *testing.B) {
+	suite := &benchmark.Suite{}
+	err := suite.Setup()
 	if err != nil {
 		b.Fatal(err)
 	}
-	benchmark.Wrapper(b, func() {
-		assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "init").Run()) //nolint:gosec // testing only
-	}, func() {
-		assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "stop", "-f").Run())   //nolint:gosec // testing only
-		assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "remove", "-f").Run()) //nolint:gosec // testing only	})
+
+	b.Run("BenchmarkVMInit", func(b *testing.B) {
+		suite.BenchmarkVMInit(b)
 	})
-}
 
-func BenchmarkVMStart(b *testing.B) {
-	subject, err := benchmark.GetSubject()
-	if err != nil {
-		b.Fatal(err)
-	}
-	assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "init").Run())       //nolint:gosec // testing only
-	assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only
-	b.ResetTimer()
-	benchmark.Wrapper(b, func() {
-		assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "start").Run()) //nolint:gosec // testing only
-	}, func() {
-		assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only
+	b.Run("BenchmarkVMStart", func(b *testing.B) {
+		suite.BenchmarkVMStart(b)
 	})
-	assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "remove", "-f").Run()) //nolint:gosec // testing only
 }