support for setup/teardown commands in cmd tests (#49)

* add ability to substitute env vars into command output/error * add support for setup/teardown commands. remove flags field in favor of just providing command string. * fix file/directory existence logging * gofmt * change build script to take entire image path instead of just tag * add back flags field * remove env vars field. hardcode replacing PWD in output/error regexes * remove flags field. change command field to a list of strings, containing flags * unify running setup/teardown/normal commands. restructure methods * update documentation * update tag check test config * remove duplicate code. fix readme * remove envvar substitution function and move PWD hack to regex compiler * remove pwd hack
GoogleContainerTools · Nov 10, 2016 · 435569b · 435569b
1 parent 1fdfd49
commit 435569b
Show file tree

Hide file tree

Showing 6 changed files with 100 additions and 66 deletions.
diff --git a/check_if_image_tag_exists/structure_test.yaml b/check_if_image_tag_exists/structure_test.yaml
@@ -1,13 +1,11 @@
 commandTests:
 - name:  'uname'
-  command: 'uname'
-  flags: '-s'
+  command: ['uname', '-s']
   expectedError: ['foo']
   expectedOutput: ['Linux\n']
 
 - name:  'broken uname'
-  command: 'uname'
-  flags: '-s -asdf'
+  command: ['uname', '-s', '-asdf']
   expectedError: ['.*[invalid | unrecognized].*']
   expectedOutput: []
 

diff --git a/structure_tests/README.md b/structure_tests/README.md
@@ -23,8 +23,9 @@ Command tests ensure that certain commands run properly on top of the shell of t
 #### Supported Fields:
 
 - Name (string, **required**): The name of the test
-- Command (string, **required**): The command to run
-- Flags (string[], *optional*): Optional list of flags to pass to the command
+- Setup ([][]string, *optional*): A list of commands (each with optional flags) to run before the actual command under test.
+- Teardown ([][]string, *optional*): A list of commands (each with optional flags) to run after the actual command under test.
+- Command ([]string, **required**): The command to run, along with the flags to pass to it.
 - Expected Output (string[], *optional*): List of regexes that should match the stdout from running the command.
 - Excluded Output (string[], *optional*): List of regexes that should **not** match the stdout from running the command.
 - Expected Error (string[], *optional*): List of regexes that should match the stderr from running the command.
@@ -34,26 +35,24 @@ Example:
 ```json
 "commandTests": [
 	{
-		"name": "apt-get",
-		"command": "apt-get",
-		"flags": ["help"],
-		"expectedOutput": [".*Usage.*"],
-		"excludedError": [".*FAIL.*"]
-	},{
 		"name": "apt-get upgrade",
-		"command": "apt-get",
-		"flags": ["-qqs", "upgrade"],
+		"command": ["apt-get", "-qqs", "upgrade"],
 		"excludedOutput": [".*Inst.*Security.* | .*Security.*Inst.*"],
 		"excludedError": [".*Inst.*Security.* | .*Security.*Inst.*"]
+	},{
+		"name": "Custom Node Version",
+		"setup": [["install_node", "v5.9.0"]],
+		"teardown": [["install_node", "v6.9.1"]],
+		"command": ["node", "-v"],
+  		"expectedOutput": ["v5.9.0\n"]
 	}
 ]
 ```
 
 ```yaml
 commandTests:
 - name:  'apt-get'
-  command: 'apt-get'
-  flags: 'help'
+  command: ['apt-get', 'help']
   expectedError: ['.*Usage.*']
   excludedError: ['*FAIL.*']
 ```

diff --git a/structure_tests/build.sh b/structure_tests/build.sh
@@ -14,13 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+usage() { echo "Usage: ./build.sh [target_image_path]"; exit 1; }
+
 set -e
 
-export VERSION=$1
+export IMAGE=$1
 
-if [ -z "$1" ]; then
-  echo "Please provide valid version to tag image."
-  exit 1
+if [ -z "$IMAGE" ]; then
+  usage
 fi
 
 envsubst < cloudbuild.yaml.in > cloudbuild.yaml

diff --git a/structure_tests/cloudbuild.yaml.in b/structure_tests/cloudbuild.yaml.in
@@ -3,6 +3,6 @@ steps:
           args: ['test', '-c', 'github.com/GoogleCloudPlatform/runtimes-common/structure_tests', '-o', '/workspace/structure_tests/structure_test']
           env: ['PROJECT_ROOT=github.com/GoogleCloudPlatform/runtimes-common']
         - name: gcr.io/cloud-builders/docker
-          args: ['build', '-t', 'gcr.io/gcp-runtimes/structure_test:${VERSION}', './structure_tests']
+          args: ['build', '-t', '${IMAGE}', './structure_tests']
 images:
-        - 'gcr.io/gcp-runtimes/structure_test:${VERSION}'
+        - '${IMAGE}'
diff --git a/structure_tests/command_test_v1.go b/structure_tests/command_test_v1.go
@@ -20,8 +20,9 @@ import (
 
 type CommandTestv1 struct {
 	Name           string
-	Command        string
-	Flags          []string
+	Setup          [][]string
+	Teardown       [][]string
+	Command        []string
 	ExpectedOutput []string
 	ExcludedOutput []string
 	ExpectedError  []string
@@ -32,7 +33,7 @@ func validateCommandTestV1(t *testing.T, tt CommandTestv1) {
 	if tt.Name == "" {
 		t.Fatalf("Please provide a valid name for every test!")
 	}
-	if tt.Command == "" {
+	if tt.Command == nil || len(tt.Command) == 0 {
 		t.Fatalf("Please provide a valid command to run for test %s", tt.Name)
 	}
 	t.Logf("COMMAND TEST: %s", tt.Name)

diff --git a/structure_tests/structure_test_v1.go b/structure_tests/structure_test_v1.go
@@ -37,48 +37,15 @@ func (st StructureTestv1) RunAll(t *testing.T) {
 func (st StructureTestv1) RunCommandTests(t *testing.T) {
 	for _, tt := range st.CommandTests {
 		validateCommandTestV1(t, tt)
-		var cmd *exec.Cmd
-		if tt.Flags != nil && len(tt.Flags) > 0 {
-			cmd = exec.Command(tt.Command, tt.Flags...)
-		} else {
-			cmd = exec.Command(tt.Command)
-		}
-		t.Logf("Executing: %s", cmd.Args)
-		var outbuf, errbuf bytes.Buffer
-
-		cmd.Stdout = &outbuf
-		cmd.Stderr = &errbuf
-
-		if err := cmd.Run(); err != nil {
-			// The test might be designed to run a command that exits with an error.
-			t.Logf("Error running command: %s. Continuing.", err)
-		}
-
-		stdout := outbuf.String()
-		if stdout != "" {
-			t.Logf("stdout: %s", stdout)
-		}
-		stderr := errbuf.String()
-		if stderr != "" {
-			t.Logf("stderr: %s", stderr)
+		for _, setup := range tt.Setup {
+			ProcessCommand(t, setup, false)
 		}
 
-		for _, errStr := range tt.ExpectedError {
-			errMsg := fmt.Sprintf("Expected string '%s' not found in error!", errStr)
-			compileAndRunRegex(errStr, stderr, t, errMsg, true)
-		}
-		for _, errStr := range tt.ExcludedError {
-			errMsg := fmt.Sprintf("Excluded string '%s' found in error!", errStr)
-			compileAndRunRegex(errStr, stderr, t, errMsg, false)
-		}
+		stdout, stderr := ProcessCommand(t, tt.Command, true)
+		CheckOutput(t, tt, stdout, stderr)
 
-		for _, outStr := range tt.ExpectedOutput {
-			errMsg := fmt.Sprintf("Expected string '%s' not found in output!", outStr)
-			compileAndRunRegex(outStr, stdout, t, errMsg, true)
-		}
-		for _, outStr := range tt.ExcludedOutput {
-			errMsg := fmt.Sprintf("Excluded string '%s' found in output!", outStr)
-			compileAndRunRegex(outStr, stdout, t, errMsg, false)
+		for _, teardown := range tt.Teardown {
+			ProcessCommand(t, teardown, false)
 		}
 	}
 }
@@ -93,9 +60,17 @@ func (st StructureTestv1) RunFileExistenceTests(t *testing.T) {
 			_, err = ioutil.ReadFile(tt.Path)
 		}
 		if tt.ShouldExist && err != nil {
-			t.Errorf("File %s should exist but does not!", tt.Path)
+			if tt.IsDirectory {
+				t.Errorf("Directory %s should exist but does not!", tt.Path)
+			} else {
+				t.Errorf("File %s should exist but does not!", tt.Path)
+			}
 		} else if !tt.ShouldExist && err == nil {
-			t.Errorf("File %s should not exist but does!", tt.Path)
+			if tt.IsDirectory {
+				t.Errorf("Directory %s should not exist but does!", tt.Path)
+			} else {
+				t.Errorf("File %s should not exist but does!", tt.Path)
+			}
 		}
 	}
 }
@@ -121,3 +96,63 @@ func (st StructureTestv1) RunFileContentTests(t *testing.T) {
 		}
 	}
 }
+
+func ProcessCommand(t *testing.T, fullCommand []string, checkOutput bool) (string, string) {
+	var cmd *exec.Cmd
+	command := fullCommand[0]
+	flags := fullCommand[1:]
+	if len(flags) > 0 {
+		cmd = exec.Command(command, flags...)
+	} else {
+		cmd = exec.Command(command)
+	}
+
+	if checkOutput {
+		t.Logf("Executing: %s", cmd.Args)
+	} else {
+		t.Logf("Executing setup/teardown: %s", cmd.Args)
+	}
+
+	var outbuf, errbuf bytes.Buffer
+
+	cmd.Stdout = &outbuf
+	cmd.Stderr = &errbuf
+
+	err := cmd.Run()
+	stdout := outbuf.String()
+	if stdout != "" {
+		t.Logf("stdout: %s", stdout)
+	}
+	stderr := errbuf.String()
+	if stderr != "" {
+		t.Logf("stderr: %s", stderr)
+	}
+	if err != nil {
+		if checkOutput {
+			// The test might be designed to run a command that exits with an error.
+			t.Logf("Error running command: %s. Continuing.", err)
+		} else {
+			t.Fatalf("Error running setup/teardown command: %s.", err)
+		}
+	}
+	return stdout, stderr
+}
+
+func CheckOutput(t *testing.T, tt CommandTestv1, stdout string, stderr string) {
+	for _, errStr := range tt.ExpectedError {
+		errMsg := fmt.Sprintf("Expected string '%s' not found in error!", errStr)
+		compileAndRunRegex(errStr, stderr, t, errMsg, true)
+	}
+	for _, errStr := range tt.ExcludedError {
+		errMsg := fmt.Sprintf("Excluded string '%s' found in error!", errStr)
+		compileAndRunRegex(errStr, stderr, t, errMsg, false)
+	}
+	for _, outStr := range tt.ExpectedOutput {
+		errMsg := fmt.Sprintf("Expected string '%s' not found in output!", outStr)
+		compileAndRunRegex(outStr, stdout, t, errMsg, true)
+	}
+	for _, outStr := range tt.ExcludedOutput {
+		errMsg := fmt.Sprintf("Excluded string '%s' found in output!", outStr)
+		compileAndRunRegex(outStr, stdout, t, errMsg, false)
+	}
+}