From 35fc0b3c920289cdfb9e7bed135a95dcabcf8c52 Mon Sep 17 00:00:00 2001 From: Christie Wilson Date: Mon, 4 May 2020 20:31:18 -0400 Subject: [PATCH] Execute yaml examples via go tests In #2540 we are seeing that some yaml tests are timing out, but it's hard to see what yaml tests are failing. This commit moves the logic out of bash and into individual go tests - now we will run an individual go test for each yaml example, completing all v1alpha1 before all v1beta1 and cleaning up in between. The output will still be challenging to read since it will be interleaved, however the failures should at least be associated with a specific yaml file. This also makes it easier to run all tests locally, though if you interrupt the tests you end up with your cluster in a bad state and it might be good to update these to execute each example in a separate namespace (in which case we could run all of v1alpha1 and v1beta1 at the same time as well!) --- examples/yaml_test.go | 241 +++++++++++++++++++++++++++++++++++++++++ go.mod | 8 +- go.sum | 36 ++++++ test/e2e-common.sh | 65 ----------- test/e2e-tests-yaml.sh | 19 +--- 5 files changed, 283 insertions(+), 86 deletions(-) create mode 100644 examples/yaml_test.go diff --git a/examples/yaml_test.go b/examples/yaml_test.go new file mode 100644 index 00000000000..625a072d4dd --- /dev/null +++ b/examples/yaml_test.go @@ -0,0 +1,241 @@ +/* +Copyright 2020 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "bytes" + "fmt" + "io/ioutil" + "path" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "os" + "os/exec" +) + +const ( + timeoutSeconds = 600 + sleepBetween = 10 +) + +// cmd will run the command c with args and if input is provided, that will be piped +// into the process as input +func cmd(t *testing.T, c string, args []string, input string) (string, error) { + t.Helper() + binary, err := exec.LookPath(c) + if err != nil { + t.Fatalf("couldn't find %s, plz install", c) + } + + t.Logf("Executing %s %v", binary, args) + + cmd := exec.Command(binary, args...) + cmd.Env = os.Environ() + + if input != "" { + cmd.Stdin = strings.NewReader(input) + } + + var stderr, stdout bytes.Buffer + cmd.Stderr = &stderr + cmd.Stdout = &stdout + if err := cmd.Run(); err != nil { + t.Fatalf("couldn't run command %s %v: %v, %s", c, args, err, stderr.String()) + return "", err + } + + return stdout.String(), nil +} + +// resetState will erase all types that the yaml examples could have concievably created. +func resetState(t *testing.T) { + t.Helper() + + crdTypes := []string{ + "conditions.tekton.dev", + "pipelineresources.tekton.dev", + "tasks.tekton.dev", + "clustertasks.tekton.dev", + "pipelines.tekton.dev", + "taskruns.tekton.dev", + "pipelineruns.tekton.dev", + "services", + "pods", + "configmaps", + "secrets", + "serviceaccounts", + "persistentvolumeclaims", + } + for _, c := range crdTypes { + cmd(t, "kubectl", []string{"delete", "--ignore-not-found=true", c, "--all"}, "") + } +} + +// getYamls will look in the directory in examples indicated by version and run for yaml files +func getYamls(t *testing.T, version, run string) []string { + t.Helper() + _, filename, _, _ := runtime.Caller(0) + + // Don't read recursively; the only dir within these dirs is no-ci which doesn't + // want any tests run against it + files, err := ioutil.ReadDir(path.Join(path.Dir(filename), version, run)) + if err != nil { + t.Fatalf("Couldn't read yaml files from %s/%s/%s: %v", path.Dir(filename), version, run, err) + } + yamls := []string{} + for _, f := range files { + if matches, _ := filepath.Match("*.yaml", f.Name()); matches { + yamls = append(yamls, f.Name()) + } + } + return yamls +} + +// replaceDockerRepo will look in the content f and replace the hard coded docker +// repo with the on provided via the KO_DOCKER_REPO environment variable +func replaceDockerRepo(t *testing.T, f string) string { + t.Helper() + r := os.Getenv("KO_DOCKER_REPO") + if r == "" { + t.Fatalf("KO_DOCKER_REPO must be set") + } + read, err := ioutil.ReadFile(f) + if err != nil { + t.Fatalf("couldnt read contents of %s: %v", f, err) + } + return strings.Replace(string(read), "gcr.io/christiewilson-catfactory", r, -1) +} + +// logRun will retrieve the entire yaml of run and log it +func logRun(t *testing.T, run string) { + t.Helper() + yaml, err := cmd(t, "kubectl", []string{"get", run, "-o", "yaml"}, "") + if err == nil { + t.Logf(yaml) + } +} + +// pollRun will use kubectl to query the specified run to see if it +// has completed. It will timeout after timeoutSeconds. +func pollRun(t *testing.T, run string, wg *sync.WaitGroup) { + t.Helper() + for i := 0; i < (timeoutSeconds / sleepBetween); i++ { + status, err := cmd(t, "kubectl", []string{"get", run, "--output=jsonpath={.status.conditions[*].status}"}, "") + if err != nil { + t.Fatalf("couldnt get status of %s: %v", run, err) + wg.Done() + return + } + + switch status { + case "", "Unknown": + // Not finished running yet + time.Sleep(sleepBetween * time.Second) + case "True": + t.Logf("%s completed successfully", run) + wg.Done() + return + default: + t.Errorf("%s has failed with status %s", run, status) + logRun(t, run) + + wg.Done() + return + } + } + t.Errorf("%s did not complete within %d seconds", run, timeoutSeconds) + logRun(t, run) + wg.Done() +} + +// waitForAllRuns will use kubectl to poll all runs in runs until completed, +// failed, or timed out +func waitForAllRuns(t *testing.T, runs []string) { + t.Helper() + + var wg sync.WaitGroup + for _, run := range runs { + wg.Add(1) + go pollRun(t, run, &wg) + } + wg.Wait() +} + +// getRuns will look for "run" in the provided ko output to determine the names +// of any runs created +func getRuns(k string) []string { + runs := []string{} + for _, s := range strings.Split(k, "\n") { + name := strings.TrimSuffix(s, " created") + if strings.Contains(name, "run") { + runs = append(runs, name) + } + } + return runs +} + +// runTests will use ko to create all yamls in the directory indicated by version +// and run, and wait for all runs (PipelineRuns and TaskRuns) created +func runTests(t *testing.T, version, run string) { + yamls := getYamls(t, version, run) + for _, yaml := range yamls { + y := yaml + + t.Run(fmt.Sprintf("%s/%s", run, y), func(t *testing.T) { + t.Parallel() + + t.Logf("Applying %s", y) + content := replaceDockerRepo(t, fmt.Sprintf("%s/%s/%s", version, run, y)) + output, err := cmd(t, "ko", []string{"create", "-f", "-"}, content) + if err == nil { + runs := getRuns(output) + + if len(runs) == 0 { + t.Fatalf("no runs were created for %s, output %s", y, output) + } + + t.Logf("Waiting for created runs %v", runs) + waitForAllRuns(t, runs) + } + }) + } +} + +func TestYaml(t *testing.T) { + versions := []string{"v1alpha1", "v1beta1"} + runs := []string{"taskruns", "pipelineruns"} + + t.Cleanup(func() { resetState(t) }) + for i, version := range versions { + t.Run(version, func(t *testing.T) { + for _, run := range runs { + runTests(t, version, run) + } + }) + // Since the v1alpha1 and v1beta1 tests are mostly copies of each other, + // we need to wipe them out between invocations + if i == 0 { + t.Logf("deleting all %s types between tests", version) + resetState(t) + } + } +} diff --git a/go.mod b/go.mod index df1b7180653..6efdf3ddc11 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.13 require ( cloud.google.com/go v0.47.0 // indirect + cloud.google.com/go/storage v1.0.0 contrib.go.opencensus.io/exporter/stackdriver v0.12.8 // indirect github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher v0.0.0-20191203181535-308b93ad1f39 github.com/cloudevents/sdk-go/v2 v2.0.0-preview8 @@ -11,8 +12,9 @@ require ( github.com/go-openapi/spec v0.19.4 // indirect github.com/golang/protobuf v1.3.3 // indirect github.com/google/go-cmp v0.4.0 - github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b + github.com/google/go-containerregistry v0.0.0-20200310013544-4fe717a9b4cb github.com/google/gofuzz v1.1.0 // indirect + github.com/google/ko v0.4.1-0.20200504014251-d45c52775002 // indirect github.com/google/uuid v1.1.1 github.com/googleapis/gnostic v0.3.1 // indirect github.com/hashicorp/go-multierror v1.0.0 @@ -23,9 +25,8 @@ require ( github.com/markbates/inflect v1.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 // indirect - github.com/onsi/ginkgo v1.10.1 // indirect github.com/onsi/gomega v1.7.0 // indirect - github.com/pkg/errors v0.8.1 + github.com/pkg/errors v0.9.1 github.com/prometheus/common v0.7.0 // indirect github.com/shurcooL/githubv4 v0.0.0-20191102174205-af46314aec7b // indirect github.com/tektoncd/plumbing v0.0.0-20200430135134-e53521e1d887 @@ -40,6 +41,7 @@ require ( golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect golang.org/x/tools v0.0.0-20200214144324-88be01311a71 // indirect gomodules.xyz/jsonpatch/v2 v2.1.0 + google.golang.org/api v0.15.0 google.golang.org/appengine v1.6.5 // indirect k8s.io/api v0.17.3 k8s.io/apiextensions-apiserver v0.17.3 // indirect diff --git a/go.sum b/go.sum index 5fa29dc5b86..be9ebffc5a3 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,7 @@ github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher v0.0.0-20191203181535-308b93ad1f39 h1:Pjo3SOZigEnIGevhFqcbFndnqyCH8WimcREd3hRM9vU= github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher v0.0.0-20191203181535-308b93ad1f39/go.mod h1:yfGmCjKuUzk9WzubMlW2zwjhCraIc/J+M40cufdemRM= github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= +github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -144,6 +145,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017 h1:2HQmlpI3yI9deH18Q6xiSOIjXD4sLI55Y/gfpa8/558= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v0.0.0-20200303162255-7d407207c304 h1:A7SYzidcyuQ/yS4wezWGYeUioUFJQk8HYWY9aMYTF4I= +github.com/docker/cli v0.0.0-20200303162255-7d407207c304/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= @@ -158,6 +161,8 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960 h1:aRd8M7HJVZOqn/vhOzrGcQH0lNAMkqMn+pXUYkatmcA= +github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -286,6 +291,8 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b h1:oGqapkPUiypdS9ch/Vu0npPe03RQ0BhVDYli+OEKNAA= github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= +github.com/google/go-containerregistry v0.0.0-20200310013544-4fe717a9b4cb h1:YZuYn5esRDuCPmdzqWupP4mM/CnOy/ZkSHEP0eXpiLA= +github.com/google/go-containerregistry v0.0.0-20200310013544-4fe717a9b4cb/go.mod h1:m8YvHwSOuBCq25yrj1DaX/fIMrv6ec3CNg8jY8+5PEA= github.com/google/go-github/v29 v29.0.3/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E= github.com/google/go-licenses v0.0.0-20200227160636-0fa8c766a591/go.mod h1:JWeTIGPLQ9gF618ZOdlUitd1gRR/l99WOkHOlmR/UVA= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -295,6 +302,8 @@ github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/ko v0.4.1-0.20200504014251-d45c52775002 h1:awXvmiTRp2Zc4TGYb1IfMuEPhTgJ3GJIVKagHu0DJgY= +github.com/google/ko v0.4.1-0.20200504014251-d45c52775002/go.mod h1:Esxyo4UQpeijNxt/11ds3rHqHLaAaHErQIdXJkUueTY= github.com/google/licenseclassifier v0.0.0-20190926221455-842c0d70d702 h1:nVgx26pAe6l/02mYomOuZssv28XkacGw/0WeiTVorqw= github.com/google/licenseclassifier v0.0.0-20190926221455-842c0d70d702/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= @@ -324,6 +333,8 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q= +github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.8.5 h1:2+KSC78XiO6Qy0hIjfc1OD9H+hsaJdJlb8Kqsd41CTE= @@ -348,6 +359,7 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= @@ -401,9 +413,12 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lightstep/tracecontext.go v0.0.0-20181129014701-1757c391b1ac h1:+2b6iGRJe3hvV/yVXrd41yVEjxuFHxasJqDhkIjS4gk= github.com/lightstep/tracecontext.go v0.0.0-20181129014701-1757c391b1ac/go.mod h1:Frd2bnT3w5FB5q49ENTfVlztJES+1k/7lyWX2+9gq/M= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -415,6 +430,8 @@ github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/markbates/inflect v1.0.4 h1:5fh1gzTFhfae06u3hzHYO9xe3l3v3nW5Pwt3naLTP5g= github.com/markbates/inflect v1.0.4/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs= +github.com/mattmoor/dep-notify v0.0.0-20190205035814-a45dec370a17 h1:P91eDVgVzvF2EmA6fmGCyR2VQFlmo2nsmS2DbHoGAco= +github.com/mattmoor/dep-notify v0.0.0-20190205035814-a45dec370a17/go.mod h1:qWnF4u+oS4UWOZMwZcBQXrt5IQIdWc6XVJLDdxGIfdQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -423,6 +440,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -458,6 +476,7 @@ github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -480,6 +499,7 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -490,6 +510,8 @@ github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -545,17 +567,22 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= @@ -770,6 +797,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= @@ -785,6 +813,7 @@ golang.org/x/tools v0.0.0-20191118222007-07fc4c7f2b98/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200115165105-de0b1760071a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200210192313-1ace956b0e17/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200214144324-88be01311a71 h1:VT2r398mD48D57f1QbzHk9R7OpTL/1UmQ7U2zqmkOuU= golang.org/x/tools v0.0.0-20200214144324-88be01311a71/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -870,8 +899,11 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d h1:LCPbGQ34PMrwad11aMZ+dbz5SAsq/0ySjRwQ8I9Qwd8= +gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -888,6 +920,8 @@ k8s.io/apimachinery v0.16.5 h1:5rnJuabGmLtgcMrdeTb48CvV0UW4TrM5vBbWWEy6OS0= k8s.io/apimachinery v0.16.5/go.mod h1:mhhO3hoLkWO+2eCvqjPtH2Ly92l9nJDwsswzWKpkN2w= k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= k8s.io/apiserver v0.17.3/go.mod h1:iJtsPpu1ZpEnHaNawpSV0nYTGBhhX2dUlnn7/QS7QiY= +k8s.io/cli-runtime v0.17.0 h1:XEuStbJBHCQlEKFyTQmceDKEWOSYHZkcYWKp3SsQ9Hk= +k8s.io/cli-runtime v0.17.0/go.mod h1:1E5iQpMODZq2lMWLUJELwRu2MLWIzwvMgDBpn3Y81Qo= k8s.io/client-go v0.16.5 h1:gp+ALoobYbhrm1CHo//8rAvyDuZqM6SDqnMT9nSSx1I= k8s.io/client-go v0.16.5/go.mod h1:0Y5GaECkDkadoJg7lBQLiQQGFl67O4Gia/dHZboA7xg= k8s.io/cloud-provider v0.17.0/go.mod h1:Ze4c3w2C0bRsjkBUoHpFi+qWe3ob1wI2/7cUn+YQIDE= @@ -921,6 +955,8 @@ modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= pack.ag/amqp v0.11.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= +sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 389d2df0130..ee6261c42e9 100755 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -62,71 +62,6 @@ function dump_extra_cluster_state() { kubectl -n tekton-pipelines logs $(get_app_pod tekton-pipelines-webhook tekton-pipelines) } -function validate_run() { - local tests_finished=0 - for i in {1..60}; do - local finished="$(kubectl get $1.tekton.dev --output=jsonpath='{.items[*].status.conditions[*].status}')" - if [[ ! "$finished" == *"Unknown"* ]]; then - tests_finished=1 - break - fi - sleep 10 - done - - return ${tests_finished} -} - -function check_results() { - local failed=0 - results="$(kubectl get $1.tekton.dev --output=jsonpath='{range .items[*]}{.metadata.name}={.status.conditions[*].type}{.status.conditions[*].status}{" "}{end}')" - for result in ${results}; do - if [[ ! "${result,,}" == *"=succeededtrue" ]]; then - echo "ERROR: test ${result} but should be succeededtrue" - failed=1 - fi - done - - return ${failed} -} - -function create_resources() { - local resource=$1 - echo ">> Creating resources ${resource}" - - # Applying the resources, either *taskruns or * *pipelineruns except those - # in the no-ci directory - for file in $(find ${REPO_ROOT_DIR}/examples/${resource}s/ -name '*.yaml' -not -path '*/no-ci/*' | sort); do - perl -p -e 's/gcr.io\/christiewilson-catfactory/$ENV{KO_DOCKER_REPO}/g' ${file} | ko create -f - || return 1 - done -} - -function run_tests() { - local resource=$1 - - # Wait for tests to finish. - echo ">> Waiting for tests to finish for ${resource}" - if validate_run $resource; then - echo "ERROR: tests timed out" - fi - - # Check that tests passed. - echo ">> Checking test results for ${resource}" - if check_results $resource; then - echo ">> All YAML tests passed" - return 0 - fi - return 1 -} - -function run_yaml_tests() { - echo ">> Starting tests for the resource ${1}/${2}" - create_resources ${1}/${2} || fail_test "Could not create ${2}/${1} from the examples" - if ! run_tests ${2}; then - return 1 - fi - return 0 -} - function install_pipeline_crd() { echo ">> Deploying Tekton Pipelines" ko resolve -f config/ \ diff --git a/test/e2e-tests-yaml.sh b/test/e2e-tests-yaml.sh index 6d2061dca9f..657d9d549b2 100755 --- a/test/e2e-tests-yaml.sh +++ b/test/e2e-tests-yaml.sh @@ -32,24 +32,7 @@ set +o pipefail install_pipeline_crd # Run the tests -failed=0 -for version in v1alpha1 v1beta1; do - for test in taskrun pipelinerun; do - header "Running YAML e2e tests for ${version} ${test}s" - if ! run_yaml_tests ${version} ${test}; then - echo "ERROR: one or more YAML tests failed" - output_yaml_test_results ${test} - output_pods_logs ${test} - failed=1 - fi - done - # Clean resources - delete_pipeline_resources - for res in services pods configmaps secrets serviceaccounts persistentvolumeclaims; do - kubectl delete --ignore-not-found=true ${res} --all - done -done - +failed=$(go test -timeout 15m ./examples) (( failed )) && fail_test success