Add acceptance test for meta policy engine and scaling actions.

The acceptance testing has been expanded to cover scaling actions
as well as the Nomad meta policy backend. The Nomad meta engine
requires Sherpa to run in a different mode than the rest of the
tests, therefore the make targets have been split to make this
clear until the whole process is automated.

The scale tests are particulary basic, but are a start for further
work in the future.

These tests have already uncovered #59.

GNUmakefile

@@ -14,10 +14,14 @@ test: ## Run the Sherpa test suite with coverage
 	@go test ./... -cover -v -tags -race \
 		"$(BUILDTAGS)" $(shell go list ./... | grep -v vendor)
 
-acctest: ## Run the Sherpa acceptance test suite
+acctest-standard: ## Run the Sherpa acceptance test suite
 	@echo "==> Running $@..."
 	@SHERPA_ACC=1 go test ./test -count 1 -v -mod vendor
 
+acctest-meta: ## Run the Sherpa Nomad meta acceptance test suite
+	@echo "==> Running $@..."
+	@SHERPA_ACC=1 SHERPA_ACC_META=1 go test ./test -count 1 -v -mod vendor
+
 release: ## Trigger the release build script
 	@echo "==> Running $@..."
 	@goreleaser --rm-dist

test/acctest/acctest.go

@@ -16,8 +16,8 @@ type TestCase struct {
 	// Steps are ran in order stopping on failure
 	Steps []TestStep
 
-	// CleanupFunc is called at the end of the TestCase if set
-	CleanupFunc TestStateFunc
+	// CleanupFuncs is called at the end of the TestCase if set
+	CleanupFuncs []TestStateFunc
 }
 
 // TestStep is a single step within a TestCase
@@ -110,8 +110,8 @@ func Test(t *testing.T, c TestCase) {
 		}
 	}
 
-	if c.CleanupFunc != nil {
-		err = c.CleanupFunc(state)
+	for i := range c.CleanupFuncs {
+		err = c.CleanupFuncs[i](state)
 		if err != nil {
 			t.Errorf("cleanup failed: %s", err)
 		}

test/acctest/helpers.go

@@ -2,12 +2,17 @@ package acctest
 
 import (
 	"fmt"
+	"time"
+
+	nomad "github.com/hashicorp/nomad/api"
 )
 
+func StringToPointer(s string) *string { return &s }
+func IntToPointer(i int) *int          { return &i }
+
 // CleanupPurgeJob is a cleanup func to purge the TestCase job from Nomad
 func CleanupPurgeJob(s *TestState) error {
 	_, _, err := s.Nomad.Jobs().Deregister(s.JobName, true, nil)
-	// TODO: wait for action to complete
 	return err
 }
 
@@ -23,6 +28,28 @@ func CheckErrEqual(expected string) func(error) bool {
 	}
 }
 
+// CheckJobReachesStatus performs a check, with a timeout that the test job reaches to desired
+// status.
+func CheckJobReachesStatus(s *TestState, status string) error {
+	timeout := time.After(30 * time.Second)
+	tick := time.Tick(500 * time.Millisecond)
+
+	for {
+		select {
+		case <-timeout:
+			return fmt.Errorf("timeout reached on checking that job %s reaches status %s", s.JobName, status)
+		case <-tick:
+			j, _, err := s.Nomad.Jobs().Info(s.JobName, nil)
+			if err != nil {
+				return err
+			}
+			if *j.Status == status {
+				return nil
+			}
+		}
+	}
+}
+
 // CheckDeploymentStatus is a TestStateFunc to check if the latest deployment of
 // the TestCase job in Nomad matches the desired status
 func CheckDeploymentStatus(status string) TestStateFunc {
@@ -61,3 +88,26 @@ func CheckTaskGroupCount(groupName string, count int) TestStateFunc {
 		return fmt.Errorf("unable to find task group %s", groupName)
 	}
 }
+
+func BuildBaseTestJob(name string) *nomad.Job {
+	return &nomad.Job{
+		ID:          StringToPointer(name),
+		Name:        StringToPointer(name),
+		Datacenters: []string{"dc1"},
+	}
+}
+
+func BuildBaseTaskGroup(group, task string) *nomad.TaskGroup {
+	return &nomad.TaskGroup{
+		Name: StringToPointer(group),
+		Tasks: []*nomad.Task{{
+			Name:   task,
+			Driver: "docker",
+			Config: map[string]interface{}{"image": "redis:3.2"},
+			Resources: &nomad.Resources{
+				CPU:      IntToPointer(500),
+				MemoryMB: IntToPointer(256),
+			},
+		}},
+	}
+}

test/policy_test.go → test/api_policy_test.go

@@ -2,13 +2,18 @@ package test
 
 import (
 	"fmt"
+	"os"
 	"testing"
 
 	"github.com/jrasell/sherpa/pkg/api"
 	"github.com/jrasell/sherpa/test/acctest"
 )
 
 func TestPolicy_list(t *testing.T) {
+	if os.Getenv("SHERPA_ACC_META") != "" {
+		t.SkipNow()
+	}
+
 	acctest.Test(t, acctest.TestCase{
 		Steps: []acctest.TestStep{
 			{
@@ -51,11 +56,15 @@ func TestPolicy_list(t *testing.T) {
 				},
 			},
 		},
-		CleanupFunc: acctest.CleanupSherpaPolicy,
+		CleanupFuncs: []acctest.TestStateFunc{acctest.CleanupSherpaPolicy},
 	})
 }
 
 func TestPolicy_readJob(t *testing.T) {
+	if os.Getenv("SHERPA_ACC_META") != "" {
+		t.SkipNow()
+	}
+
 	groupName := "group"
 
 	acctest.Test(t, acctest.TestCase{
@@ -93,11 +102,15 @@ func TestPolicy_readJob(t *testing.T) {
 				},
 			},
 		},
-		CleanupFunc: acctest.CleanupSherpaPolicy,
+		CleanupFuncs: []acctest.TestStateFunc{acctest.CleanupSherpaPolicy},
 	})
 }
 
 func TestPolicy_readJobGroup(t *testing.T) {
+	if os.Getenv("SHERPA_ACC_META") != "" {
+		t.SkipNow()
+	}
+
 	groupName := "group"
 
 	acctest.Test(t, acctest.TestCase{
@@ -134,11 +147,15 @@ func TestPolicy_readJobGroup(t *testing.T) {
 				},
 			},
 		},
-		CleanupFunc: acctest.CleanupSherpaPolicy,
+		CleanupFuncs: []acctest.TestStateFunc{acctest.CleanupSherpaPolicy},
 	})
 }
 
 func TestPolicy_write(t *testing.T) {
+	if os.Getenv("SHERPA_ACC_META") != "" {
+		t.SkipNow()
+	}
+
 	groupName := "group"
 
 	acctest.Test(t, acctest.TestCase{
@@ -198,11 +215,15 @@ func TestPolicy_write(t *testing.T) {
 				},
 			},
 		},
-		CleanupFunc: acctest.CleanupSherpaPolicy,
+		CleanupFuncs: []acctest.TestStateFunc{acctest.CleanupSherpaPolicy},
 	})
 }
 
 func TestPolicy_deleteJobPolicy(t *testing.T) {
+	if os.Getenv("SHERPA_ACC_META") != "" {
+		t.SkipNow()
+	}
+
 	groupName := "group"
 
 	acctest.Test(t, acctest.TestCase{
@@ -252,11 +273,15 @@ func TestPolicy_deleteJobPolicy(t *testing.T) {
 				CheckErr:  acctest.CheckErrEqual("unexpected response code 404: 404 page not found"),
 			},
 		},
-		CleanupFunc: acctest.CleanupSherpaPolicy,
+		CleanupFuncs: []acctest.TestStateFunc{acctest.CleanupSherpaPolicy},
 	})
 }
 
 func TestPolicy_deleteJobGroupPolicy(t *testing.T) {
+	if os.Getenv("SHERPA_ACC_META") != "" {
+		t.SkipNow()
+	}
+
 	acctest.Test(t, acctest.TestCase{
 		Steps: []acctest.TestStep{
 			{
@@ -326,6 +351,6 @@ func TestPolicy_deleteJobGroupPolicy(t *testing.T) {
 				},
 			},
 		},
-		CleanupFunc: acctest.CleanupSherpaPolicy,
+		CleanupFuncs: []acctest.TestStateFunc{acctest.CleanupSherpaPolicy},
 	})
 }