diff --git a/Makefile b/Makefile index edf5540e8..914eb9118 100644 --- a/Makefile +++ b/Makefile @@ -268,6 +268,10 @@ test-e2e-container: test-e2e-vm: go test -ldflags $(LDFLAGS) -timeout 45m ./e2e/vm -test.v -ginkgo.v --installed="$(INSTALLED)" +.PHONY: test-benchmark +test-benchmark: + cd benchmark/all && go test -ldflags $(LDFLAGS) -bench=. -benchmem --installed="$(INSTALLED)" + .PHONY: test-benchmark-vm test-benchmark-vm: cd benchmark/vm && go test -ldflags $(LDFLAGS) -bench=. -benchmem --installed="$(INSTALLED)" diff --git a/benchmark/all/all_test.go b/benchmark/all/all_test.go new file mode 100644 index 000000000..1ba257d8d --- /dev/null +++ b/benchmark/all/all_test.go @@ -0,0 +1,49 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package all runs all the benchmark tests of Finch. +package all + +import ( + "testing" + + "github.com/runfinch/finch/benchmark" +) + +func BenchmarkAll(b *testing.B) { + suite := &benchmark.Suite{} + err := suite.Setup() + if err != nil { + b.Fatal(err) + } + + b.Run("BenchmarkVMInit", func(b *testing.B) { + suite.BenchmarkVMInit(b) + }) + + b.Run("BenchmarkVMStart", func(b *testing.B) { + suite.BenchmarkVMStart(b) + }) + + err = suite.InitVM() + if err != nil { + b.Fatal(err) + } + + b.Run("BenchmarkContainerRun", func(b *testing.B) { + suite.BenchmarkContainerRun(b) + }) + + b.Run("BenchmarkImageBuild", func(b *testing.B) { + suite.BenchmarkImageBuild(b) + }) + + err = suite.StopVM() + if err != nil { + b.Fatal(err) + } + err = suite.RemoveVM() + if err != nil { + b.Fatal(err) + } +} diff --git a/benchmark/container/container_test.go b/benchmark/container/container_test.go index 926fba8db..cc84d7432 100644 --- a/benchmark/container/container_test.go +++ b/benchmark/container/container_test.go @@ -5,57 +5,23 @@ package container import ( - "fmt" - "os" - "os/exec" - "path/filepath" "testing" - "github.com/stretchr/testify/assert" - "github.com/runfinch/finch/benchmark" ) -const ( - virtualMachineRootCmd = "vm" - alpineImage = "public.ecr.aws/docker/library/alpine:latest" - testImageName = "test:tag" - testContainerName = "ctr-test" -) - -func BenchmarkContainerRun(b *testing.B) { - subject, err := benchmark.GetSubject() +func BenchmarkContainer(b *testing.B) { + suite := &benchmark.Suite{} + err := suite.Setup() if err != nil { b.Fatal(err) } - assert.NoError(b, exec.Command(subject, "pull", alpineImage).Run()) //nolint:gosec // testing only - benchmark.Wrapper(b, func() { - assert.NoError(b, exec.Command(subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only - }, func() { - assert.NoError(b, exec.Command(subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only + + b.Run("BenchmarkContainerRun", func(b *testing.B) { + suite.BenchmarkContainerRun(b) }) - assert.NoError(b, exec.Command(subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only -} -func BenchmarkImageBuild(b *testing.B) { - subject, err := benchmark.GetSubject() - if err != nil { - b.Fatal(err) - } - homeDir, err := os.UserHomeDir() - assert.NoError(b, err) - tempDir, err := os.MkdirTemp(homeDir, "finch-test") - assert.NoError(b, err) - dockerFilePath := filepath.Join(tempDir, "Dockerfile") - err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s - CMD ["echo", "finch-test-dummy-output"] - `, alpineImage)), 0o644) - assert.NoError(b, err) - buildContext := filepath.Dir(dockerFilePath) - defer os.RemoveAll(buildContext) //nolint:errcheck // testing only - benchmark.Wrapper(b, func() { - assert.NoError(b, exec.Command(subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only - }, func() { - assert.NoError(b, exec.Command(subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only + b.Run("BenchmarkImageBuild", func(b *testing.B) { + suite.BenchmarkImageBuild(b) }) } diff --git a/benchmark/suite.go b/benchmark/suite.go new file mode 100644 index 000000000..1a49d8176 --- /dev/null +++ b/benchmark/suite.go @@ -0,0 +1,111 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package benchmark runs benchmark tests of Finch. +package benchmark + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + virtualMachineRootCmd = "vm" + alpineImage = "public.ecr.aws/docker/library/alpine:latest" + testImageName = "test:tag" + testContainerName = "ctr-test" +) + +// Suite is a struct that groups benchmark functions and shared state. +type Suite struct { + subject string +} + +// Setup initializes the Suite by getting the subject. +func (suite *Suite) Setup() error { + subject, err := GetSubject() + if err != nil { + return err + } + suite.subject = subject + return nil +} + +// InitVM initializes a virtual machine. +func (suite *Suite) InitVM() error { + return exec.Command(suite.subject, virtualMachineRootCmd, "init").Run() // #nosec G204 +} + +// StartVM starts the VM. +func (suite *Suite) StartVM() error { + return exec.Command(suite.subject, virtualMachineRootCmd, "start", "-f").Run() // #nosec G204 +} + +// StopVM stops the VM. +func (suite *Suite) StopVM() error { + return exec.Command(suite.subject, virtualMachineRootCmd, "stop", "-f").Run() // #nosec G204 +} + +// RemoveVM removes the VM. +func (suite *Suite) RemoveVM() error { + return exec.Command(suite.subject, virtualMachineRootCmd, "remove", "-f").Run() // #nosec G204 +} + +// BenchmarkVMInit measures the metrics to initialize the VM. +func (suite *Suite) BenchmarkVMInit(b *testing.B) { + Wrapper(b, func() { + assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "init").Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only + assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "remove", "-f").Run()) //nolint:gosec // testing only }) + }) +} + +// BenchmarkVMStart measures the metrics to start the VM. +func (suite *Suite) BenchmarkVMStart(b *testing.B) { + assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "init").Run()) //nolint:gosec // testing only + assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only + b.ResetTimer() + Wrapper(b, func() { + assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "start").Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only + }) + assert.NoError(b, exec.Command(suite.subject, virtualMachineRootCmd, "remove", "-f").Run()) //nolint:gosec // testing only +} + +// BenchmarkContainerRun measures the metrics to run a container. +func (suite *Suite) BenchmarkContainerRun(b *testing.B) { + assert.NoError(b, exec.Command(suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only + Wrapper(b, func() { + assert.NoError(b, exec.Command(suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command(suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only + }) + assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only +} + +// BenchmarkImageBuild measures the metrics to build an image. +func (suite *Suite) BenchmarkImageBuild(b *testing.B) { + homeDir, err := os.UserHomeDir() + assert.NoError(b, err) + tempDir, err := os.MkdirTemp(homeDir, "finch-test") + assert.NoError(b, err) + dockerFilePath := filepath.Join(tempDir, "Dockerfile") + err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s + CMD ["echo", "finch-test-dummy-output"] + `, alpineImage)), 0o644) + assert.NoError(b, err) + buildContext := filepath.Dir(dockerFilePath) + defer os.RemoveAll(buildContext) //nolint:errcheck // testing only + Wrapper(b, func() { + assert.NoError(b, exec.Command(suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only + }) +} diff --git a/benchmark/vm/vm_test.go b/benchmark/vm/vm_test.go index 2b62278c5..e43f28225 100644 --- a/benchmark/vm/vm_test.go +++ b/benchmark/vm/vm_test.go @@ -5,43 +5,23 @@ package vm import ( - "os/exec" "testing" - "github.com/stretchr/testify/assert" - "github.com/runfinch/finch/benchmark" ) -const ( - virtualMachineRootCmd = "vm" -) - -func BenchmarkVMInit(b *testing.B) { - subject, err := benchmark.GetSubject() +func BenchmarkVM(b *testing.B) { + suite := &benchmark.Suite{} + err := suite.Setup() if err != nil { b.Fatal(err) } - benchmark.Wrapper(b, func() { - assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "init").Run()) //nolint:gosec // testing only - }, func() { - assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only - assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "remove", "-f").Run()) //nolint:gosec // testing only }) + + b.Run("BenchmarkVMInit", func(b *testing.B) { + suite.BenchmarkVMInit(b) }) -} -func BenchmarkVMStart(b *testing.B) { - subject, err := benchmark.GetSubject() - if err != nil { - b.Fatal(err) - } - assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "init").Run()) //nolint:gosec // testing only - assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only - b.ResetTimer() - benchmark.Wrapper(b, func() { - assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "start").Run()) //nolint:gosec // testing only - }, func() { - assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "stop", "-f").Run()) //nolint:gosec // testing only + b.Run("BenchmarkVMStart", func(b *testing.B) { + suite.BenchmarkVMStart(b) }) - assert.NoError(b, exec.Command(subject, virtualMachineRootCmd, "remove", "-f").Run()) //nolint:gosec // testing only }