test: refactor to benchmark suite and add test-benchmark target

Makefile

@@ -268,6 +268,10 @@ test-e2e-container:
 test-e2e-vm:
 	go test -ldflags $(LDFLAGS) -timeout 45m ./e2e/vm -test.v -ginkgo.v --installed="$(INSTALLED)"
 
+.PHONY: test-benchmark
+test-benchmark:
+	cd benchmark/all && go test -ldflags $(LDFLAGS) -bench=. -benchmem --installed="$(INSTALLED)"
+
 .PHONY: test-benchmark-vm
 test-benchmark-vm:
 	cd benchmark/vm && go test -ldflags $(LDFLAGS) -bench=. -benchmem --installed="$(INSTALLED)"

benchmark/all/all_test.go

@@ -0,0 +1,49 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package all runs all the benchmark tests of Finch.
+package all
+
+import (
+	"testing"
+
+	"github.com/runfinch/finch/benchmark"
+)
+
+func BenchmarkAll(b *testing.B) {
+	suite := &benchmark.Suite{}
+	err := suite.Setup()
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.Run("BenchmarkVMInit", func(b *testing.B) {
+		suite.BenchmarkVMInit(b)
+	})
+
+	b.Run("BenchmarkVMStart", func(b *testing.B) {
+		suite.BenchmarkVMStart(b)
+	})
+
+	err = suite.InitVM()
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.Run("BenchmarkContainerRun", func(b *testing.B) {
+		suite.BenchmarkContainerRun(b)
+	})
+
+	b.Run("BenchmarkImageBuild", func(b *testing.B) {
+		suite.BenchmarkImageBuild(b)
+	})
+
+	err = suite.StopVM()
+	if err != nil {
+		b.Fatal(err)
+	}
+	err = suite.RemoveVM()
+	if err != nil {
+		b.Fatal(err)
+	}
+}

benchmark/container/container_test.go

@@ -5,57 +5,23 @@
 package container
 
 import (
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/runfinch/finch/benchmark"
 )
 
-const (
-	virtualMachineRootCmd = "vm"
-	alpineImage           = "public.ecr.aws/docker/library/alpine:latest"
-	testImageName         = "test:tag"
-	testContainerName     = "ctr-test"
-)
-
-func BenchmarkContainerRun(b *testing.B) {
-	subject, err := benchmark.GetSubject()
+func BenchmarkContainer(b *testing.B) {
+	suite := &benchmark.Suite{}
+	err := suite.Setup()
 	if err != nil {
 		b.Fatal(err)
 	}
-	assert.NoError(b, exec.Command(subject, "pull", alpineImage).Run()) //nolint:gosec // testing only
-	benchmark.Wrapper(b, func() {
-		assert.NoError(b, exec.Command(subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
-	}, func() {
-		assert.NoError(b, exec.Command(subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
+
+	b.Run("BenchmarkContainerRun", func(b *testing.B) {
+		suite.BenchmarkContainerRun(b)
 	})
-	assert.NoError(b, exec.Command(subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
-}
 
-func BenchmarkImageBuild(b *testing.B) {
-	subject, err := benchmark.GetSubject()
-	if err != nil {
-		b.Fatal(err)
-	}
-	homeDir, err := os.UserHomeDir()
-	assert.NoError(b, err)
-	tempDir, err := os.MkdirTemp(homeDir, "finch-test")
-	assert.NoError(b, err)
-	dockerFilePath := filepath.Join(tempDir, "Dockerfile")
-	err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s
-			CMD ["echo", "finch-test-dummy-output"]
-			`, alpineImage)), 0o644)
-	assert.NoError(b, err)
-	buildContext := filepath.Dir(dockerFilePath)
-	defer os.RemoveAll(buildContext) //nolint:errcheck // testing only
-	benchmark.Wrapper(b, func() {
-		assert.NoError(b, exec.Command(subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
-	}, func() {
-		assert.NoError(b, exec.Command(subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
+	b.Run("BenchmarkImageBuild", func(b *testing.B) {
+		suite.BenchmarkImageBuild(b)
 	})
 }

benchmark/suite.go

@@ -0,0 +1,111 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package benchmark runs benchmark tests of Finch.
+package benchmark
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	virtualMachineRootCmd = "vm"
+	alpineImage           = "public.ecr.aws/docker/library/alpine:latest"
+	testImageName         = "test:tag"
+	testContainerName     = "ctr-test"
+)
+
+// Suite is a struct that groups benchmark functions and shared state.
+type Suite struct {
+	subject string
+}
+
+// Setup initializes the Suite by getting the subject.
+func (suite *Suite) Setup() error {
+	subject, err := GetSubject()
+	if err != nil {
+		return err
+	}
+	suite.subject = subject
+	return nil
+}
+
+// InitVM initializes a virtual machine.
+func (suite *Suite) InitVM() error {
+	return exec.Command(suite.subject, virtualMachineRootCmd, "init").Run() // #nosec G204
+}
+
+// StartVM starts the VM.
+func (suite *Suite) StartVM() error {
+	return exec.Command(suite.subject, virtualMachineRootCmd, "start", "-f").Run() // #nosec G204
+}
+
+// StopVM stops the VM.
+func (suite *Suite) StopVM() error {
+	return exec.Command(suite.subject, virtualMachineRootCmd, "stop", "-f").Run() // #nosec G204
+}
+
+// RemoveVM removes the VM.
+func (suite *Suite) RemoveVM() error {
+	return exec.Command(suite.subject, virtualMachineRootCmd, "remove", "-f").Run() // #nosec G204
+}
+
+// BenchmarkVMInit measures the metrics to initialize the VM.
+func (suite *Suite) BenchmarkVMInit(b *testing.B) {
+	Wrapper(b, func() {
+		assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "init").Run()) //nolint:gosec // testing only
+	}, func() {
+		assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "stop", "-f").Run())   //nolint:gosec // testing only
+		assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "remove", "-f").Run()) //nolint:gosec // testing only	})
+	})
+}
+
+// BenchmarkVMStart measures the metrics to start the VM.
+func (suite *Suite) BenchmarkVMStart(b *testing.B) {
+	assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "init").Run())       //nolint:gosec // testing only
+	assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only
+	b.ResetTimer()
+	Wrapper(b, func() {
+		assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "start").Run()) //nolint:gosec // testing only
+	}, func() {
+		assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only
+	})
+	assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "remove", "-f").Run()) //nolint:gosec // testing only
+}
+
+// BenchmarkContainerRun measures the metrics to run a container.
+func (suite *Suite) BenchmarkContainerRun(b *testing.B) {
+	assert.NoError(b, exec.Command(suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only
+	Wrapper(b, func() {
+		assert.NoError(b, exec.Command(suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
+	}, func() {
+		assert.NoError(b, exec.Command(suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
+	})
+	assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
+}
+
+// BenchmarkImageBuild measures the metrics to build an image.
+func (suite *Suite) BenchmarkImageBuild(b *testing.B) {
+	homeDir, err := os.UserHomeDir()
+	assert.NoError(b, err)
+	tempDir, err := os.MkdirTemp(homeDir, "finch-test")
+	assert.NoError(b, err)
+	dockerFilePath := filepath.Join(tempDir, "Dockerfile")
+	err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s
+			CMD ["echo", "finch-test-dummy-output"]
+			`, alpineImage)), 0o644)
+	assert.NoError(b, err)
+	buildContext := filepath.Dir(dockerFilePath)
+	defer os.RemoveAll(buildContext) //nolint:errcheck // testing only
+	Wrapper(b, func() {
+		assert.NoError(b, exec.Command(suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
+	}, func() {
+		assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
+	})
+}

benchmark/vm/vm_test.go

@@ -5,43 +5,23 @@
 package vm
 
 import (
-	"os/exec"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/runfinch/finch/benchmark"
 )
 
-const (
-	virtualMachineRootCmd = "vm"
-)
-
-func BenchmarkVMInit(b *testing.B) {
-	subject, err := benchmark.GetSubject()
+func BenchmarkVM(b *testing.B) {
+	suite := &benchmark.Suite{}
+	err := suite.Setup()
 	if err != nil {
 		b.Fatal(err)
 	}
-	benchmark.Wrapper(b, func() {
-		assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "init").Run()) //nolint:gosec // testing only
-	}, func() {
-		assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "stop", "-f").Run())   //nolint:gosec // testing only
-		assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "remove", "-f").Run()) //nolint:gosec // testing only	})
+
+	b.Run("BenchmarkVMInit", func(b *testing.B) {
+		suite.BenchmarkVMInit(b)
 	})
-}
 
-func BenchmarkVMStart(b *testing.B) {
-	subject, err := benchmark.GetSubject()
-	if err != nil {
-		b.Fatal(err)
-	}
-	assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "init").Run())       //nolint:gosec // testing only
-	assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only
-	b.ResetTimer()
-	benchmark.Wrapper(b, func() {
-		assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "start").Run()) //nolint:gosec // testing only
-	}, func() {
-		assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only
+	b.Run("BenchmarkVMStart", func(b *testing.B) {
+		suite.BenchmarkVMStart(b)
 	})
-	assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "remove", "-f").Run()) //nolint:gosec // testing only
 }