From 0c590d15c9489e0ae21aa2560a8893ab76ddf08f Mon Sep 17 00:00:00 2001 From: Christie Wilson Date: Mon, 4 May 2020 20:31:18 -0400 Subject: [PATCH 1/2] Execute yaml examples via go tests In #2540 we are seeing that some yaml tests are timing out, but it's hard to see what yaml tests are failing. This commit moves the logic out of bash and into individual go tests - now we will run an individual go test for each yaml example, completing all v1alpha1 before all v1beta1 and cleaning up in between. The output will still be challenging to read since it will be interleaved, however the failures should at least be associated with a specific yaml file. This also makes it easier to run all tests locally, though if you interrupt the tests you end up with your cluster in a bad state and it might be good to update these to execute each example in a separate namespace (in which case we could run all of v1alpha1 and v1beta1 at the same time as well!) --- examples/examples_test.go | 261 ++++++++++++++++++ .../pipelineruns/clustertask-pipelinerun.yaml | 4 +- examples/v1alpha1/taskruns/clustertask.yaml | 4 +- .../optional-resources-with-clustertask.yaml | 4 +- .../pipelineruns/clustertask-pipelinerun.yaml | 4 +- examples/v1beta1/taskruns/clustertask.yaml | 4 +- .../optional-resources-with-clustertask.yaml | 4 +- test/README.md | 11 +- test/e2e-common.sh | 65 ----- test/e2e-tests-yaml.sh | 20 +- 10 files changed, 283 insertions(+), 98 deletions(-) create mode 100644 examples/examples_test.go diff --git a/examples/examples_test.go b/examples/examples_test.go new file mode 100644 index 00000000000..44228e5be61 --- /dev/null +++ b/examples/examples_test.go @@ -0,0 +1,261 @@ +// +build examples + +/* +Copyright 2020 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "bytes" + "fmt" + "io/ioutil" + "path" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "os" + "os/exec" + + "github.com/tektoncd/pipeline/pkg/names" + knativetest "knative.dev/pkg/test" +) + +const ( + timeoutSeconds = 600 + sleepBetween = 10 + + // we may want to consider either not running examples that require registry access + // or doing something more sophisticated to inject the right registry in when folks + // are executing the examples + horribleHardCodedRegistry = "gcr.io/christiewilson-catfactory" +) + +// formatLogger is a printf style function for logging in tests. +type formatLogger func(template string, args ...interface{}) + +// cmd will run the command c with args and if input is provided, that will be piped +// into the process as input +func cmd(logf formatLogger, c string, args []string, input string) (string, error) { + // Remove braces from args when logging so users can see the complete call + logf("Executing %s %v", c, strings.Trim(fmt.Sprint(args), "[]")) + + cmd := exec.Command(c, args...) + cmd.Env = os.Environ() + + if input != "" { + cmd.Stdin = strings.NewReader(input) + } + + var stderr, stdout bytes.Buffer + cmd.Stderr = &stderr + cmd.Stdout = &stdout + if err := cmd.Run(); err != nil { + logf("couldn't run command %s %v: %v, %s", c, args, err, stderr.String()) + return "", err + } + + return stdout.String(), nil +} + +// eraseClusterTasks will erase all cluster tasks which do not get cleaned up by namespace. +func eraseClusterTasks(logf formatLogger) { + if _, err := cmd(logf, "kubectl", []string{"delete", "--ignore-not-found=true", "clustertasks.tekton.dev", "--all"}, ""); err != nil { + logf("couldn't delete cluster tasks: %v", err) + } +} + +// getYamls will look in the directory in examples indicated by version and run for yaml files +func getYamls(t *testing.T, version, run string) []string { + t.Helper() + _, filename, _, _ := runtime.Caller(0) + + // Don't read recursively; the only dir within these dirs is no-ci which doesn't + // want any tests run against it + files, err := ioutil.ReadDir(path.Join(path.Dir(filename), version, run)) + if err != nil { + t.Fatalf("Couldn't read yaml files from %s/%s/%s: %v", path.Dir(filename), version, run, err) + } + yamls := []string{} + for _, f := range files { + if matches, _ := filepath.Match("*.yaml", f.Name()); matches { + yamls = append(yamls, f.Name()) + } + } + return yamls +} + +// replaceDockerRepo will look in the content f and replace the hard coded docker +// repo with the on provided via the KO_DOCKER_REPO environment variable +func replaceDockerRepo(t *testing.T, f string) string { + t.Helper() + r := os.Getenv("KO_DOCKER_REPO") + if r == "" { + t.Fatalf("KO_DOCKER_REPO must be set") + } + read, err := ioutil.ReadFile(f) + if err != nil { + t.Fatalf("couldnt read contents of %s: %v", f, err) + } + return strings.Replace(string(read), horribleHardCodedRegistry, r, -1) +} + +// logRun will retrieve the entire yaml of run in namespace n and log it +func logRun(t *testing.T, n, run string) { + t.Helper() + yaml, err := cmd(t.Logf, "kubectl", []string{"--namespace", n, "get", run, "-o", "yaml"}, "") + if err == nil { + t.Logf(yaml) + } +} + +// pollRun will use kubectl to query the specified run in namespace n +// to see if it has completed. It will timeout after timeoutSeconds. +func pollRun(t *testing.T, n, run string, wg *sync.WaitGroup) { + t.Helper() + // instead of polling it might be faster to explore using --watch and parsing + // the output as it returns + for i := 0; i < (timeoutSeconds / sleepBetween); i++ { + status, err := cmd(t.Logf, "kubectl", []string{"--namespace", n, "get", run, "--output=jsonpath={.status.conditions[*].status}"}, "") + if err != nil { + t.Fatalf("couldnt get status of %s: %v", run, err) + wg.Done() + return + } + + switch status { + case "", "Unknown": + // Not finished running yet + time.Sleep(sleepBetween * time.Second) + case "True": + t.Logf("%s completed successfully", run) + wg.Done() + return + default: + t.Errorf("%s has failed with status %s", run, status) + logRun(t, n, run) + + wg.Done() + return + } + } + t.Errorf("%s did not complete within %d seconds", run, timeoutSeconds) + logRun(t, n, run) + wg.Done() +} + +// waitForAllRuns will use kubectl to poll all runs in runs in namespace n +// until completed, failed, or timed out +func waitForAllRuns(t *testing.T, n string, runs []string) { + t.Helper() + + var wg sync.WaitGroup + for _, run := range runs { + wg.Add(1) + go pollRun(t, n, run, &wg) + } + wg.Wait() +} + +// getRuns will look for "run" in the provided ko output to determine the names +// of any runs created. +func getRuns(k string) []string { + runs := []string{} + // this a pretty naive way of looking for these run names, it would be an + // improvement to be more aware of the output format or to parse the yamls and + // use a client to apply them so we can get the name of the created runs via + // the client + for _, s := range strings.Split(k, "\n") { + name := strings.TrimSuffix(s, " created") + if strings.Contains(name, "run") { + runs = append(runs, name) + } + } + return runs +} + +// createRandomNamespace will create a namespace with a randomized name so that anything +// happening within this namespace will not conflict with other namespaces. It will return +// the name of the namespace. +func createRandomNamespace(t *testing.T) string { + t.Helper() + + n := names.SimpleNameGenerator.RestrictLengthWithRandomSuffix("irkalla") + _, err := cmd(t.Logf, "kubectl", []string{"create", "namespace", n}, "") + if err != nil { + t.Fatalf("could not create namespace %s for example: %v", n, err) + } + return n +} + +// deleteNamespace will delete the namespace with kubectl. +func deleteNamespace(logf formatLogger, n string) { + _, err := cmd(logf, "kubectl", []string{"delete", "namespace", n}, "") + if err != nil { + logf("could not delete namespace %s for example: %v", n, err) + } +} + +// runTests will use ko to create all yamls in the directory indicated by version +// and run, and wait for all runs (PipelineRuns and TaskRuns) created +func runTests(t *testing.T, version, run string) { + yamls := getYamls(t, version, run) + for _, yaml := range yamls { + y := yaml + + t.Run(fmt.Sprintf("%s/%s", run, y), func(t *testing.T) { + t.Parallel() + + // apply the yaml in its own namespace so it can run in parallel + // with similar tests (e.g. v1alpha1 + v1beta1) without conflicting with each other + // and we can easily cleanup. + n := createRandomNamespace(t) + knativetest.CleanupOnInterrupt(func() { deleteNamespace(t.Logf, n) }, t.Logf) + defer deleteNamespace(t.Logf, n) + + t.Logf("Applying %s to namespace %s", y, n) + content := replaceDockerRepo(t, fmt.Sprintf("%s/%s/%s", version, run, y)) + output, err := cmd(t.Logf, "ko", []string{"create", "--namespace", n, "-f", "-"}, content) + if err == nil { + runs := getRuns(output) + + if len(runs) == 0 { + t.Fatalf("no runs were created for %s, output %s", y, output) + } + + t.Logf("Waiting for created runs %v", runs) + waitForAllRuns(t, n, runs) + } + }) + } +} + +func TestYaml(t *testing.T) { + versions := []string{"v1alpha1", "v1beta1"} + runs := []string{"taskruns", "pipelineruns"} + + knativetest.CleanupOnInterrupt(func() { eraseClusterTasks(t.Logf) }, t.Logf) + defer eraseClusterTasks(t.Logf) + + for _, version := range versions { + for _, run := range runs { + runTests(t, version, run) + } + } +} diff --git a/examples/v1alpha1/pipelineruns/clustertask-pipelinerun.yaml b/examples/v1alpha1/pipelineruns/clustertask-pipelinerun.yaml index 9c1818d73e7..c9bf0d776af 100644 --- a/examples/v1alpha1/pipelineruns/clustertask-pipelinerun.yaml +++ b/examples/v1alpha1/pipelineruns/clustertask-pipelinerun.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1alpha1 kind: ClusterTask metadata: - name: cluster-task-pipeline-4 + name: cluster-task-pipeline-4-v1alpha1 spec: steps: - name: task-two-step-one @@ -17,7 +17,7 @@ spec: tasks: - name: cluster-task-pipeline-4 taskRef: - name: cluster-task-pipeline-4 + name: cluster-task-pipeline-4-v1alpha1 kind: ClusterTask --- apiVersion: tekton.dev/v1alpha1 diff --git a/examples/v1alpha1/taskruns/clustertask.yaml b/examples/v1alpha1/taskruns/clustertask.yaml index 3e3e1f8f844..876f5a35f6d 100644 --- a/examples/v1alpha1/taskruns/clustertask.yaml +++ b/examples/v1alpha1/taskruns/clustertask.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1alpha1 kind: ClusterTask metadata: - name: clustertask + name: clustertask-v1alpha1 spec: steps: - image: ubuntu @@ -13,5 +13,5 @@ metadata: generateName: clustertask- spec: taskRef: - name: clustertask + name: clustertask-v1alpha1 kind: ClusterTask diff --git a/examples/v1alpha1/taskruns/optional-resources-with-clustertask.yaml b/examples/v1alpha1/taskruns/optional-resources-with-clustertask.yaml index 428e857e94d..53ede3d6979 100644 --- a/examples/v1alpha1/taskruns/optional-resources-with-clustertask.yaml +++ b/examples/v1alpha1/taskruns/optional-resources-with-clustertask.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1alpha1 kind: ClusterTask metadata: - name: clustertask-with-optional-resources + name: clustertask-with-optional-resources-v1alpha1 spec: inputs: resources: @@ -31,5 +31,5 @@ metadata: name: clustertask-without-resources spec: taskRef: - name: clustertask-with-optional-resources + name: clustertask-with-optional-resources-v1alpha1 kind: ClusterTask diff --git a/examples/v1beta1/pipelineruns/clustertask-pipelinerun.yaml b/examples/v1beta1/pipelineruns/clustertask-pipelinerun.yaml index d70960dbb12..78314a28126 100644 --- a/examples/v1beta1/pipelineruns/clustertask-pipelinerun.yaml +++ b/examples/v1beta1/pipelineruns/clustertask-pipelinerun.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1beta1 kind: ClusterTask metadata: - name: cluster-task-pipeline-4 + name: cluster-task-pipeline-4-v1beta1 spec: steps: - name: task-two-step-one @@ -17,7 +17,7 @@ spec: tasks: - name: cluster-task-pipeline-4 taskRef: - name: cluster-task-pipeline-4 + name: cluster-task-pipeline-4-v1beta1 kind: ClusterTask --- apiVersion: tekton.dev/v1beta1 diff --git a/examples/v1beta1/taskruns/clustertask.yaml b/examples/v1beta1/taskruns/clustertask.yaml index 10972427dab..9124271adc9 100644 --- a/examples/v1beta1/taskruns/clustertask.yaml +++ b/examples/v1beta1/taskruns/clustertask.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1beta1 kind: ClusterTask metadata: - name: clustertask + name: clustertask-v1beta1 spec: steps: - image: ubuntu @@ -13,5 +13,5 @@ metadata: generateName: clustertask- spec: taskRef: - name: clustertask + name: clustertask-v1beta1 kind: ClusterTask diff --git a/examples/v1beta1/taskruns/optional-resources-with-clustertask.yaml b/examples/v1beta1/taskruns/optional-resources-with-clustertask.yaml index 1ecae87cf8d..882443c6971 100644 --- a/examples/v1beta1/taskruns/optional-resources-with-clustertask.yaml +++ b/examples/v1beta1/taskruns/optional-resources-with-clustertask.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1beta1 kind: ClusterTask metadata: - name: clustertask-with-optional-resources + name: clustertask-with-optional-resources-v1beta1 spec: params: - name: filename @@ -30,5 +30,5 @@ metadata: name: clustertask-without-resources spec: taskRef: - name: clustertask-with-optional-resources + name: clustertask-with-optional-resources-v1beta1 kind: ClusterTask diff --git a/test/README.md b/test/README.md index 5b135f929e8..e6bc5c3c49b 100644 --- a/test/README.md +++ b/test/README.md @@ -201,12 +201,17 @@ TestTaskRun, use go test -v -tags=e2e -count=1 ./test -run ^TestTaskRun ``` -### Running YAML tests +### Running example YAML tests -To run the YAML e2e tests, run the following command: +To run the tests that exercise [the examples](../examples), you can run the tests +with go. To run these tests, you must provide `go` with `-tags=examples`. The tests +run against your current kubeconfig. + + +For example: ```bash -./test/e2e-tests-yaml.sh +go test -v -count=1 -tags=examples -timeout 15m ./examples ``` ### Running upgrade tests diff --git a/test/e2e-common.sh b/test/e2e-common.sh index e0a6328bc33..ee6261c42e9 100755 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -62,71 +62,6 @@ function dump_extra_cluster_state() { kubectl -n tekton-pipelines logs $(get_app_pod tekton-pipelines-webhook tekton-pipelines) } -function validate_run() { - local tests_finished=0 - for i in {1..90}; do - local finished="$(kubectl get $1.tekton.dev --output=jsonpath='{.items[*].status.conditions[*].status}')" - if [[ ! "$finished" == *"Unknown"* ]]; then - tests_finished=1 - break - fi - sleep 10 - done - - return ${tests_finished} -} - -function check_results() { - local failed=0 - results="$(kubectl get $1.tekton.dev --output=jsonpath='{range .items[*]}{.metadata.name}={.status.conditions[*].type}{.status.conditions[*].status}{" "}{end}')" - for result in ${results}; do - if [[ ! "${result,,}" == *"=succeededtrue" ]]; then - echo "ERROR: test ${result} but should be succeededtrue" - failed=1 - fi - done - - return ${failed} -} - -function create_resources() { - local resource=$1 - echo ">> Creating resources ${resource}" - - # Applying the resources, either *taskruns or * *pipelineruns except those - # in the no-ci directory - for file in $(find ${REPO_ROOT_DIR}/examples/${resource}s/ -name '*.yaml' -not -path '*/no-ci/*' | sort); do - perl -p -e 's/gcr.io\/christiewilson-catfactory/$ENV{KO_DOCKER_REPO}/g' ${file} | ko create -f - || return 1 - done -} - -function run_tests() { - local resource=$1 - - # Wait for tests to finish. - echo ">> Waiting for tests to finish for ${resource}" - if validate_run $resource; then - echo "ERROR: tests timed out" - fi - - # Check that tests passed. - echo ">> Checking test results for ${resource}" - if check_results $resource; then - echo ">> All YAML tests passed" - return 0 - fi - return 1 -} - -function run_yaml_tests() { - echo ">> Starting tests for the resource ${1}/${2}" - create_resources ${1}/${2} || fail_test "Could not create ${2}/${1} from the examples" - if ! run_tests ${2}; then - return 1 - fi - return 0 -} - function install_pipeline_crd() { echo ">> Deploying Tekton Pipelines" ko resolve -f config/ \ diff --git a/test/e2e-tests-yaml.sh b/test/e2e-tests-yaml.sh index 6d2061dca9f..e83637982d9 100755 --- a/test/e2e-tests-yaml.sh +++ b/test/e2e-tests-yaml.sh @@ -32,24 +32,8 @@ set +o pipefail install_pipeline_crd # Run the tests -failed=0 -for version in v1alpha1 v1beta1; do - for test in taskrun pipelinerun; do - header "Running YAML e2e tests for ${version} ${test}s" - if ! run_yaml_tests ${version} ${test}; then - echo "ERROR: one or more YAML tests failed" - output_yaml_test_results ${test} - output_pods_logs ${test} - failed=1 - fi - done - # Clean resources - delete_pipeline_resources - for res in services pods configmaps secrets serviceaccounts persistentvolumeclaims; do - kubectl delete --ignore-not-found=true ${res} --all - done -done - +set -x +failed=$(go test -v -count=1 -tags=examples -timeout 15m ./examples) (( failed )) && fail_test success From a9206d9992a38e0b2b8a1123c5faf7c5b866c652 Mon Sep 17 00:00:00 2001 From: Christie Wilson Date: Thu, 21 May 2020 18:04:17 -0400 Subject: [PATCH 2/2] =?UTF-8?q?Add=20table=20of=20contents=20to=20test=20R?= =?UTF-8?q?EADME=20=F0=9F=93=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There's some good stuff in this doc but it's hard to remember what's in it cuz it's kinda all over the place - maybe a TOC will help! --- test/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/README.md b/test/README.md index e6bc5c3c49b..2bbb82f09da 100644 --- a/test/README.md +++ b/test/README.md @@ -13,6 +13,23 @@ go test ./... go test -v -count=1 -tags=e2e -timeout=20m ./test ``` +* [Unit tests](#unit-tests) + * [Unit testing controllers](#unit-testing-controllers) +* [End to end tests](#end-to-end-tests) + * [Setup](#setup) + * [Running](#running) + * [Flags](#flags) + * [Running specific test cases](#running-specific-test-cases) + * [Running example yaml tests](#running-example-yaml-tests) + * [Running upgrade tests](#running-upgrade-tests) + * [Adding integration tests](#adding-integration-tests) + * [Create Tekton objects](#create-tekton-objects) + * [Get access to client objects](#get-access-to-client-objects) + * [Generate random names](#generate-random-names) + * [Poll pipeline resources](#poll-pipeline-resources) +* [Presubmit tests](#presubmit-tests) + * [Running presubmit integration tests](#running-presubmit-integration-tests) + ## Unit tests Unit tests live side by side with the code they are testing and can be run with: