diff --git a/pkg/reconciler/pipelinerun/pipelinerun.go b/pkg/reconciler/pipelinerun/pipelinerun.go index c549bad7f3d..820748df266 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun.go +++ b/pkg/reconciler/pipelinerun/pipelinerun.go @@ -230,17 +230,6 @@ func (c *Reconciler) Reconcile(ctx context.Context, key string) error { return merr } -func (c *Reconciler) getPipelineFunc(tr *v1alpha1.PipelineRun) resources.GetPipeline { - var gtFunc resources.GetPipeline = func(name string) (v1alpha1.PipelineInterface, error) { - p, err := c.pipelineLister.Pipelines(tr.Namespace).Get(name) - if err != nil { - return nil, err - } - return p, nil - } - return gtFunc -} - func (c *Reconciler) updatePipelineResults(ctx context.Context, pr *v1alpha1.PipelineRun) { if err := pr.ConvertTo(ctx, &v1beta1.PipelineRun{}); err != nil { if ce, ok := err.(*v1beta1.CannotConvertError); ok { @@ -249,8 +238,12 @@ func (c *Reconciler) updatePipelineResults(ctx context.Context, pr *v1alpha1.Pip return } - getPipelineFunc := c.getPipelineFunc(pr) - pipelineMeta, pipelineSpec, err := resources.GetPipelineData(ctx, pr, getPipelineFunc) + // TODO: Use factory func instead of hard-coding this once OCI images are supported. + resolver := &resources.LocalPipelineRefResolver{ + Namespace: pr.Namespace, + Tektonclient: c.PipelineClientSet, + } + pipelineMeta, pipelineSpec, err := resources.GetPipelineData(ctx, pr, resolver.GetPipeline) if err != nil { if ce, ok := err.(*v1beta1.CannotConvertError); ok { pr.Status.MarkResourceNotConvertible(ce) @@ -341,8 +334,12 @@ func (c *Reconciler) reconcile(ctx context.Context, pr *v1alpha1.PipelineRun) er return err } - getPipelineFunc := c.getPipelineFunc(pr) - pipelineMeta, pipelineSpec, err := resources.GetPipelineData(ctx, pr, getPipelineFunc) + // TODO: Use factory func instead of hard-coding this once OCI images are supported. + resolver := &resources.LocalPipelineRefResolver{ + Namespace: pr.Namespace, + Tektonclient: c.PipelineClientSet, + } + pipelineMeta, pipelineSpec, err := resources.GetPipelineData(ctx, pr, resolver.GetPipeline) if err != nil { if ce, ok := err.(*v1beta1.CannotConvertError); ok { pr.Status.MarkResourceNotConvertible(ce) diff --git a/pkg/reconciler/pipelinerun/pipelinerun_test.go b/pkg/reconciler/pipelinerun/pipelinerun_test.go index 6c798c592f5..b45ba4201ee 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun_test.go +++ b/pkg/reconciler/pipelinerun/pipelinerun_test.go @@ -219,7 +219,7 @@ func TestReconcile(t *testing.T) { } // Check that the expected TaskRun was created - actual := clients.Pipeline.Actions()[0].(ktesting.CreateAction).GetObject() + actual := clients.Pipeline.Actions()[1].(ktesting.CreateAction).GetObject() expectedTaskRun := tb.TaskRun("test-pipeline-run-success-unit-test-1-mz4c7", tb.TaskRunNamespace("foo"), tb.TaskRunOwnerReference("PipelineRun", "test-pipeline-run-success", @@ -340,7 +340,7 @@ func TestReconcile_PipelineSpecTaskSpec(t *testing.T) { } // Check that the expected TaskRun was created - actual := clients.Pipeline.Actions()[0].(ktesting.CreateAction).GetObject() + actual := clients.Pipeline.Actions()[1].(ktesting.CreateAction).GetObject() expectedTaskRun := tb.TaskRun("test-pipeline-run-success-unit-test-task-spec-9l9zj", tb.TaskRunNamespace("foo"), tb.TaskRunOwnerReference("PipelineRun", "test-pipeline-run-success", @@ -812,19 +812,20 @@ func TestReconcileOnCompletedPipelineRun(t *testing.T) { t.Fatalf("Error reconciling: %s", err) } - if len(clients.Pipeline.Actions()) != 1 { + if len(clients.Pipeline.Actions()) != 2 { t.Fatalf("Expected client to have updated the TaskRun status for a completed PipelineRun, but it did not") } - actual := clients.Pipeline.Actions()[0].(ktesting.UpdateAction).GetObject().(*v1alpha1.PipelineRun) + actual := clients.Pipeline.Actions()[1].(ktesting.UpdateAction).GetObject().(*v1alpha1.PipelineRun) if actual == nil { t.Errorf("Expected a PipelineRun to be updated, but it wasn't.") } + t.Log(clients.Pipeline.Actions()) actions := clients.Pipeline.Actions() for _, action := range actions { if action != nil { resource := action.GetResource().Resource - if resource != "pipelineruns" { + if resource == "taskruns" { t.Fatalf("Expected client to not have created a TaskRun for the completed PipelineRun, but it did") } } @@ -959,7 +960,7 @@ func TestReconcileWithTimeout(t *testing.T) { } // Check that the expected TaskRun was created - actual := clients.Pipeline.Actions()[0].(ktesting.CreateAction).GetObject().(*v1alpha1.TaskRun) + actual := clients.Pipeline.Actions()[1].(ktesting.CreateAction).GetObject().(*v1alpha1.TaskRun) if actual == nil { t.Fatalf("Expected a TaskRun to be created, but it wasn't.") } @@ -1121,7 +1122,7 @@ func TestReconcilePropagateLabels(t *testing.T) { } // Check that the expected TaskRun was created - actual := clients.Pipeline.Actions()[0].(ktesting.CreateAction).GetObject().(*v1alpha1.TaskRun) + actual := clients.Pipeline.Actions()[1].(ktesting.CreateAction).GetObject().(*v1alpha1.TaskRun) if actual == nil { t.Errorf("Expected a TaskRun to be created, but it wasn't.") } @@ -1352,7 +1353,7 @@ func TestReconcilePropagateAnnotations(t *testing.T) { } // Check that the expected TaskRun was created - actual := clients.Pipeline.Actions()[0].(ktesting.CreateAction).GetObject().(*v1alpha1.TaskRun) + actual := clients.Pipeline.Actions()[1].(ktesting.CreateAction).GetObject().(*v1alpha1.TaskRun) if actual == nil { t.Errorf("Expected a TaskRun to be created, but it wasn't.") } @@ -1540,8 +1541,8 @@ func TestReconcileWithConditionChecks(t *testing.T) { } // Check that the expected TaskRun was created - condCheck0 := clients.Pipeline.Actions()[0].(ktesting.CreateAction).GetObject().(*v1alpha1.TaskRun) - condCheck1 := clients.Pipeline.Actions()[1].(ktesting.CreateAction).GetObject().(*v1alpha1.TaskRun) + condCheck0 := clients.Pipeline.Actions()[1].(ktesting.CreateAction).GetObject().(*v1alpha1.TaskRun) + condCheck1 := clients.Pipeline.Actions()[2].(ktesting.CreateAction).GetObject().(*v1alpha1.TaskRun) if condCheck0 == nil || condCheck1 == nil { t.Errorf("Expected two ConditionCheck TaskRuns to be created, but it wasn't.") } @@ -1646,7 +1647,7 @@ func TestReconcileWithFailingConditionChecks(t *testing.T) { } // Check that the expected TaskRun was created - actual := clients.Pipeline.Actions()[0].(ktesting.CreateAction).GetObject().(*v1alpha1.TaskRun) + actual := clients.Pipeline.Actions()[1].(ktesting.CreateAction).GetObject().(*v1alpha1.TaskRun) if actual == nil { t.Errorf("Expected a ConditionCheck TaskRun to be created, but it wasn't.") } diff --git a/pkg/reconciler/pipelinerun/resources/pipelineref.go b/pkg/reconciler/pipelinerun/resources/pipelineref.go new file mode 100644 index 00000000000..5b572b2743f --- /dev/null +++ b/pkg/reconciler/pipelinerun/resources/pipelineref.go @@ -0,0 +1,41 @@ +/* +Copyright 2020 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "fmt" + + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" + clientset "github.com/tektoncd/pipeline/pkg/client/clientset/versioned" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// LocalPipelineRefResolver uses the current cluster to resolve a pipeline reference. +type LocalPipelineRefResolver struct { + Namespace string + Tektonclient clientset.Interface +} + +// GetPipeline will resolve a Pipeline from the local cluster using a versioned Tekton client. It will +// return an error if it can't find an appropriate Pipeline for any reason. +func (l *LocalPipelineRefResolver) GetPipeline(name string) (v1alpha1.PipelineInterface, error) { + // If we are going to resolve this reference locally, we need a namespace scope. + if l.Namespace == "" { + return nil, fmt.Errorf("Must specify namespace to resolve reference to pipeline %s", name) + } + return l.Tektonclient.TektonV1alpha1().Pipelines(l.Namespace).Get(name, metav1.GetOptions{}) +} diff --git a/pkg/reconciler/pipelinerun/resources/pipelineref_test.go b/pkg/reconciler/pipelinerun/resources/pipelineref_test.go new file mode 100644 index 00000000000..606d5c67403 --- /dev/null +++ b/pkg/reconciler/pipelinerun/resources/pipelineref_test.go @@ -0,0 +1,82 @@ +/* + Copyright 2020 The Tekton Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package resources_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" + "github.com/tektoncd/pipeline/pkg/client/clientset/versioned/fake" + "github.com/tektoncd/pipeline/pkg/reconciler/pipelinerun/resources" + tb "github.com/tektoncd/pipeline/test/builder" + "k8s.io/apimachinery/pkg/runtime" +) + +func TestPipelineRef(t *testing.T) { + testcases := []struct { + name string + pipelines []runtime.Object + ref *v1alpha1.PipelineRef + expected runtime.Object + wantErr bool + }{ + { + name: "local-pipeline", + pipelines: []runtime.Object{ + tb.Pipeline("simple", tb.PipelineNamespace("default")), + tb.Pipeline("dummy", tb.PipelineNamespace("default")), + }, + ref: &v1alpha1.PipelineRef{ + Name: "simple", + }, + expected: tb.Pipeline("simple", tb.PipelineNamespace("default")), + wantErr: false, + }, + { + name: "pipeline-not-found", + pipelines: []runtime.Object{}, + ref: &v1alpha1.PipelineRef{ + Name: "simple", + }, + expected: nil, + wantErr: true, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + tektonclient := fake.NewSimpleClientset(tc.pipelines...) + + lc := &resources.LocalPipelineRefResolver{ + Namespace: "default", + Tektonclient: tektonclient, + } + + task, err := lc.GetPipeline(tc.ref.Name) + if tc.wantErr && err == nil { + t.Fatal("Expected error but found nil instead") + } else if !tc.wantErr && err != nil { + t.Fatalf("Received unexpected error ( %#v )", err) + } + + if diff := cmp.Diff(task, tc.expected); tc.expected != nil && diff != "" { + t.Error(diff) + } + }) + } +} diff --git a/pkg/reconciler/taskrun/resources/taskref.go b/pkg/reconciler/taskrun/resources/taskref.go new file mode 100644 index 00000000000..bc5ab51f19f --- /dev/null +++ b/pkg/reconciler/taskrun/resources/taskref.go @@ -0,0 +1,50 @@ +/* +Copyright 2020 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "fmt" + + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" + clientset "github.com/tektoncd/pipeline/pkg/client/clientset/versioned" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// LocalTaskRefResolver uses the current cluster to resolve a task reference. +type LocalTaskRefResolver struct { + Namespace string + Kind v1alpha1.TaskKind + Tektonclient clientset.Interface +} + +// GetTask will resolve either a Task or ClusterTask from the local cluster using a versioned Tekton client. It will +// return an error if it can't find an appropriate Task for any reason. +func (l *LocalTaskRefResolver) GetTask(name string) (v1alpha1.TaskInterface, error) { + if l.Kind == v1alpha1.ClusterTaskKind { + task, err := l.Tektonclient.TektonV1alpha1().ClusterTasks().Get(name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return task, nil + } + + // If we are going to resolve this reference locally, we need a namespace scope. + if l.Namespace == "" { + return nil, fmt.Errorf("Must specify namespace to resolve reference to task %s", name) + } + return l.Tektonclient.TektonV1alpha1().Tasks(l.Namespace).Get(name, metav1.GetOptions{}) +} diff --git a/pkg/reconciler/taskrun/resources/taskref_test.go b/pkg/reconciler/taskrun/resources/taskref_test.go new file mode 100644 index 00000000000..fc21ae66e76 --- /dev/null +++ b/pkg/reconciler/taskrun/resources/taskref_test.go @@ -0,0 +1,97 @@ +/* + Copyright 2020 The Tekton Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package resources_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" + "github.com/tektoncd/pipeline/pkg/client/clientset/versioned/fake" + "github.com/tektoncd/pipeline/pkg/reconciler/taskrun/resources" + tb "github.com/tektoncd/pipeline/test/builder" +) + +func TestTaskRef(t *testing.T) { + testcases := []struct { + name string + tasks []runtime.Object + ref *v1alpha1.TaskRef + expected runtime.Object + wantErr bool + }{ + { + name: "local-task", + tasks: []runtime.Object{ + tb.Task("simple", tb.TaskNamespace("default")), + tb.Task("dummy", tb.TaskNamespace("default")), + }, + ref: &v1alpha1.TaskRef{ + Name: "simple", + }, + expected: tb.Task("simple", tb.TaskNamespace("default")), + wantErr: false, + }, + { + name: "local-clustertask", + tasks: []runtime.Object{ + tb.ClusterTask("cluster-task"), + tb.ClusterTask("dummy-task"), + }, + ref: &v1alpha1.TaskRef{ + Name: "cluster-task", + Kind: "ClusterTask", + }, + expected: tb.ClusterTask("cluster-task"), + wantErr: false, + }, + { + name: "task-not-found", + tasks: []runtime.Object{}, + ref: &v1alpha1.TaskRef{ + Name: "simple", + }, + expected: nil, + wantErr: true, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + tektonclient := fake.NewSimpleClientset(tc.tasks...) + + lc := &resources.LocalTaskRefResolver{ + Namespace: "default", + Kind: tc.ref.Kind, + Tektonclient: tektonclient, + } + + task, err := lc.GetTask(tc.ref.Name) + if tc.wantErr && err == nil { + t.Fatal("Expected error but found nil instead") + } else if !tc.wantErr && err != nil { + t.Fatalf("Received unexpected error ( %#v )", err) + } + + if diff := cmp.Diff(task, tc.expected); tc.expected != nil && diff != "" { + t.Error(diff) + } + }) + } +} diff --git a/pkg/reconciler/taskrun/taskrun.go b/pkg/reconciler/taskrun/taskrun.go index fb6d33e87fe..079dcaf824b 100644 --- a/pkg/reconciler/taskrun/taskrun.go +++ b/pkg/reconciler/taskrun/taskrun.go @@ -208,6 +208,19 @@ func (c *Reconciler) Reconcile(ctx context.Context, key string) error { return multierror.Append(err, c.updateStatusLabelsAndAnnotations(tr, original)).ErrorOrNil() } +func (c *Reconciler) getTaskResolver(tr *v1alpha1.TaskRun) (*resources.LocalTaskRefResolver, v1alpha1.TaskKind) { + resolver := &resources.LocalTaskRefResolver{ + Namespace: tr.Namespace, + Tektonclient: c.PipelineClientSet, + } + kind := v1alpha1.NamespacedTaskKind + if tr.Spec.TaskRef != nil && tr.Spec.TaskRef.Kind == v1alpha1.ClusterTaskKind { + kind = v1alpha1.ClusterTaskKind + } + resolver.Kind = kind + return resolver, kind +} + // `prepare` fetches resources the taskrun depends on, runs validation and convertion // It may report errors back to Reconcile, it updates the taskrun status in case of // error but it does not sync updates back to etcd. It does not emit events. @@ -231,8 +244,8 @@ func (c *Reconciler) prepare(ctx context.Context, tr *v1alpha1.TaskRun) (*v1alph return nil, nil, err } - getTaskFunc, kind := c.getTaskFunc(tr) - taskMeta, taskSpec, err := resources.GetTaskData(ctx, tr, getTaskFunc) + resolver, kind := c.getTaskResolver(tr) + taskMeta, taskSpec, err := resources.GetTaskData(ctx, tr, resolver.GetTask) if err != nil { if ce, ok := err.(*v1beta1.CannotConvertError); ok { tr.Status.MarkResourceNotConvertible(ce) @@ -458,30 +471,6 @@ func (c *Reconciler) updateLabelsAndAnnotations(tr *v1alpha1.TaskRun) (*v1alpha1 return newTr, nil } -func (c *Reconciler) getTaskFunc(tr *v1alpha1.TaskRun) (resources.GetTask, v1alpha1.TaskKind) { - var gtFunc resources.GetTask - kind := v1alpha1.NamespacedTaskKind - if tr.Spec.TaskRef != nil && tr.Spec.TaskRef.Kind == v1alpha1.ClusterTaskKind { - gtFunc = func(name string) (v1alpha1.TaskInterface, error) { - t, err := c.PipelineClientSet.TektonV1alpha1().ClusterTasks().Get(name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - return t, nil - } - kind = v1alpha1.ClusterTaskKind - } else { - gtFunc = func(name string) (v1alpha1.TaskInterface, error) { - t, err := c.PipelineClientSet.TektonV1alpha1().Tasks(tr.Namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - return t, nil - } - } - return gtFunc, kind -} - func (c *Reconciler) handlePodCreationError(tr *v1alpha1.TaskRun, err error) { var msg string if isExceededResourceQuotaError(err) { diff --git a/pkg/remote/oci.go b/pkg/remote/oci.go deleted file mode 100644 index 6bdc4789466..00000000000 --- a/pkg/remote/oci.go +++ /dev/null @@ -1,109 +0,0 @@ -/* -Copyright 2019 The Tekton Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package remote - -import ( - "fmt" - "io/ioutil" - "strings" - - "github.com/google/go-containerregistry/pkg/authn" - imgname "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" - "github.com/tektoncd/pipeline/pkg/client/clientset/versioned/scheme" -) - -// KeychainProvider is an input to the OCIResolver which returns a keychain for fetching remote images with -// authentication. -type KeychainProvider func() (authn.Keychain, error) - -// OCIResolver will attempt to fetch Tekton resources from an OCI compliant image repository. -type OCIResolver struct { - imageReference string - keychainProvider KeychainProvider -} - -// GetTask will retrieve the specified task from the resolver's defined image and return its spec. If it cannot be -// retrieved for any reason, an error is returned. -func (o OCIResolver) GetTask(taskName string) (*v1beta1.TaskSpec, error) { - taskContents, err := o.readImageLayer("task", taskName) - if err != nil { - return nil, err - } - - // Deserialize the task into a valid task spec. - var task v1beta1.Task - _, _, err = scheme.Codecs.UniversalDeserializer().Decode(taskContents, nil, &task) - if err != nil { - return nil, fmt.Errorf("Invalid remote task %s: %w", taskName, err) - } - - return &task.Spec, nil -} - -func (o OCIResolver) readImageLayer(kind string, name string) ([]byte, error) { - imgRef, err := imgname.ParseReference(o.imageReference) - if err != nil { - return nil, fmt.Errorf("%s is an unparseable task image reference: %w", o.imageReference, err) - } - - // Create a keychain for use in authenticating against the remote repository. - keychain, err := o.keychainProvider() - if err != nil { - return nil, err - } - - img, err := remote.Image(imgRef, remote.WithAuthFromKeychain(keychain)) - if err != nil { - return nil, fmt.Errorf("Error pulling image %q: %w", o.imageReference, err) - } - // Ensure the media type is exclusively the Tekton catalog media type. - if mt, err := img.MediaType(); err != nil || string(mt) != "application/vnd.cdf.tekton.catalog.v1beta1+yaml" { - return nil, fmt.Errorf("cannot parse reference from image type %s: %w", string(mt), err) - } - - m, err := img.Manifest() - if err != nil { - return nil, err - } - - ls, err := img.Layers() - if err != nil { - return nil, err - } - var layer v1.Layer - for idx, l := range m.Layers { - if l.Annotations["org.opencontainers.image.title"] == o.getLayerName(kind, name) { - layer = ls[idx] - } - } - if layer == nil { - return nil, fmt.Errorf("Resource %s/%s not found", kind, name) - } - rc, err := layer.Uncompressed() - if err != nil { - return nil, err - } - defer rc.Close() - return ioutil.ReadAll(rc) -} - -func (o OCIResolver) getLayerName(kind string, name string) string { - return fmt.Sprintf("%s/%s", strings.ToLower(kind), name) -} diff --git a/pkg/remote/oci/resolver.go b/pkg/remote/oci/resolver.go new file mode 100644 index 00000000000..b95d2aaaf55 --- /dev/null +++ b/pkg/remote/oci/resolver.go @@ -0,0 +1,115 @@ +/* +Copyright 2020 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oci + +import ( + "fmt" + "io/ioutil" + + "github.com/google/go-containerregistry/pkg/authn" + imgname "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + ociremote "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/tektoncd/pipeline/pkg/client/clientset/versioned/scheme" + "github.com/tektoncd/pipeline/pkg/remote" + "k8s.io/apimachinery/pkg/runtime" +) + +var _ remote.Resolver = (*Resolver)(nil) + +// Resolver implements the Resolver interface using OCI images. +type Resolver struct { + imageReference string + keychain authn.Keychain +} + +func (o *Resolver) List() ([]remote.ResolvedObject, error) { + img, err := o.retrieveImage() + if err != nil { + return nil, err + } + + manifest, err := img.Manifest() + if err != nil { + return nil, fmt.Errorf("Could not parse image manifest: %w", err) + } + contents := make([]remote.ResolvedObject, 0, len(manifest.Layers)) + for _, l := range manifest.Layers { + contents = append(contents, remote.ResolvedObject{ + Kind: l.Annotations["cdf.tekton.image.kind"], + APIVersion: l.Annotations["cdf.tekton.image.apiVersion"], + Name: l.Annotations["org.opencontainers.image.title"], + }) + } + + return contents, nil +} + +func (o *Resolver) Get(kind, name string) (runtime.Object, error) { + img, err := o.retrieveImage() + if err != nil { + return nil, err + } + + manifest, err := img.Manifest() + if err != nil { + return nil, fmt.Errorf("Could not parse image manifest: %w", err) + } + + layers, err := img.Layers() + if err != nil { + return nil, fmt.Errorf("Could not read image layers: %w", err) + } + + for idx, l := range manifest.Layers { + lKind := l.Annotations["cdf.tekton.image.kind"] + lName := l.Annotations["org.opencontainers.image.title"] + + if kind == lKind && name == lName { + return readLayer(layers[idx]) + } + } + return nil, fmt.Errorf("Could not find object in image with kind: %s and name: %s", kind, name) +} + +// retrieveImage will fetch the image's contents and manifest. +func (o *Resolver) retrieveImage() (v1.Image, error) { + imgRef, err := imgname.ParseReference(o.imageReference) + if err != nil { + return nil, fmt.Errorf("%s is an unparseable image reference: %w", o.imageReference, err) + } + return ociremote.Image(imgRef, ociremote.WithAuthFromKeychain(o.keychain)) +} + +// Utility function to read out the contents of an image layer as a parsed Tekton resource. +func readLayer(layer v1.Layer) (runtime.Object, error) { + rc, err := layer.Uncompressed() + if err != nil { + return nil, fmt.Errorf("Failed to read image layer: %w", err) + } + defer rc.Close() + + contents, err := ioutil.ReadAll(rc) + if err != nil { + return nil, fmt.Errorf("Could not read contents of image layer: %w", err) + } + + runtime.NewScheme() + + obj, _, err := scheme.Codecs.UniversalDeserializer().Decode(contents, nil, nil) + return obj, err +} diff --git a/pkg/remote/oci/resolver_test.go b/pkg/remote/oci/resolver_test.go new file mode 100644 index 00000000000..0038620382f --- /dev/null +++ b/pkg/remote/oci/resolver_test.go @@ -0,0 +1,118 @@ +/* +Copyright 2020 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oci + +import ( + "fmt" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/tektoncd/pipeline/pkg/remote" + "github.com/tektoncd/pipeline/test" + tb "github.com/tektoncd/pipeline/test/builder" + "k8s.io/apimachinery/pkg/runtime" +) + +func TestOCIResolver(t *testing.T) { + // Set up a fake registry to push an image to. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + testcases := []struct { + name string + objs []runtime.Object + listExpected []remote.ResolvedObject + }{ + { + name: "single-task", + objs: []runtime.Object{ + tb.Task("simple-task", tb.TaskType()), + }, + listExpected: []remote.ResolvedObject{{Kind: "task", APIVersion: "v1alpha1", Name: "simple-task"}}, + }, + { + name: "cluster-task", + objs: []runtime.Object{ + tb.ClusterTask("simple-task", tb.ClusterTaskType()), + }, + listExpected: []remote.ResolvedObject{{Kind: "clustertask", APIVersion: "v1alpha1", Name: "simple-task"}}, + }, + { + name: "multiple-tasks", + objs: []runtime.Object{ + tb.Task("first-task", tb.TaskType()), + tb.Task("second-task", tb.TaskType()), + }, + listExpected: []remote.ResolvedObject{ + {Kind: "task", APIVersion: "v1alpha1", Name: "first-task"}, + {Kind: "task", APIVersion: "v1alpha1", Name: "second-task"}, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + // Create a new image with the objects. + ref, err := test.CreateImage(fmt.Sprintf("%s/testociresolve/%s", u.Host, tc.name), tc.objs...) + if err != nil { + t.Fatalf("could not push image: %#v", err) + } + + resolver := Resolver{ + imageReference: ref, + keychain: authn.DefaultKeychain, + } + + listActual, err := resolver.List() + if err != nil { + t.Errorf("unexpected error listing contents of image: %#v", err) + } + + // The contents of the image are in a specific order so we can expect this iteration to be consistent. + for idx, actual := range listActual { + if diff := cmp.Diff(actual, tc.listExpected[idx]); diff != "" { + t.Error(diff) + } + } + + for _, obj := range tc.objs { + actual, err := resolver.Get(strings.ToLower(obj.GetObjectKind().GroupVersionKind().Kind), getObjectName(obj)) + if err != nil { + t.Fatalf("could not retrieve object from image: %#v", err) + } + + if diff := cmp.Diff(actual, obj); diff != "" { + t.Error(diff) + } + } + }) + } +} + +func getObjectName(obj runtime.Object) string { + return reflect.Indirect(reflect.ValueOf(obj)).FieldByName("ObjectMeta").FieldByName("Name").String() +} diff --git a/pkg/remote/oci_test.go b/pkg/remote/oci_test.go deleted file mode 100644 index 95c52bb8072..00000000000 --- a/pkg/remote/oci_test.go +++ /dev/null @@ -1,124 +0,0 @@ -/* -Copyright 2019 The Tekton Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package remote - -import ( - "fmt" - "net/http/httptest" - "net/url" - "strings" - "testing" - - "github.com/ghodss/yaml" - "github.com/google/go-cmp/cmp" - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/registry" - imgv1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/mutate" - remoteimg "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/google/go-containerregistry/pkg/v1/tarball" - "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func pushImage(imgRef name.Reference, task *v1beta1.Task) (imgv1.Image, error) { - taskRaw, err := yaml.Marshal(task) - if err != nil { - return nil, fmt.Errorf("invalid sample task def %s", err.Error()) - } - - img := mutate.MediaType(empty.Image, types.MediaType("application/vnd.cdf.tekton.catalog.v1beta1+yaml")) - layer, err := tarball.LayerFromReader(strings.NewReader(string(taskRaw))) - if err != nil { - return nil, fmt.Errorf("unexpected error adding task layer to image %s", err.Error()) - } - - img, err = mutate.Append(img, mutate.Addendum{ - Layer: layer, - Annotations: map[string]string{ - "org.opencontainers.image.title": fmt.Sprintf("task/%s", task.GetName()), - }, - }) - if err != nil { - return nil, fmt.Errorf("could not add layer to image %s", err.Error()) - } - - if err := remoteimg.Write(imgRef, img); err != nil { - return nil, fmt.Errorf("could not push example image to registry") - } - - return img, nil -} - -func TestOCIResolver(t *testing.T) { - // Set up a fake registry to push an image to. - s := httptest.NewServer(registry.New()) - defer s.Close() - u, err := url.Parse(s.URL) - if err != nil { - t.Fatal(err) - } - - imgRef, err := name.ParseReference(fmt.Sprintf("%s/test/ociresolver", u.Host)) - if err != nil { - t.Errorf("undexpected error producing image reference %s", err.Error()) - } - - // Create the image using an example task. - task := v1beta1.Task{ - ObjectMeta: metav1.ObjectMeta{ - Name: "hello-world", - }, - Spec: v1beta1.TaskSpec{ - Steps: []v1beta1.Step{ - { - Container: v1.Container{ - Image: "ubuntu", - }, - Script: "echo \"Hello World!\"", - }, - }, - }, - } - img, err := pushImage(imgRef, &task) - if err != nil { - t.Error(err) - } - - // Now we can call our resolver and see if the spec returned is the same. - digest, err := img.Digest() - if err != nil { - t.Errorf("unexpected error getting digest of image: %s", err.Error()) - } - resolver := OCIResolver{ - imageReference: imgRef.Context().Digest(digest.String()).String(), - keychainProvider: func() (authn.Keychain, error) { return authn.DefaultKeychain, nil }, - } - - actual, err := resolver.GetTask("hello-world") - if err != nil { - t.Errorf("failed to fetch task hello-world: %s", err.Error()) - } - - if diff := cmp.Diff(actual, &task.Spec); diff != "" { - t.Error(diff) - } -} diff --git a/pkg/remote/resolver.go b/pkg/remote/resolver.go index 2e8afcb195c..7a8bb443bfb 100644 --- a/pkg/remote/resolver.go +++ b/pkg/remote/resolver.go @@ -1,12 +1,9 @@ /* Copyright 2019 The Tekton Authors - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,22 +14,20 @@ limitations under the License. package remote import ( - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/authn/k8schain" - "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + "k8s.io/apimachinery/pkg/runtime" ) -// Resolver will retrieve Tekton resources like Tasks from remote repositories like an OCI image repositories. -type Resolver interface { - GetTask(taskName string) (*v1beta1.TaskSpec, error) +// ResolvedObject is returned by Resolver.List representing a Tekton resource stored in a remote location. +type ResolvedObject struct { + Kind string + APIVersion string + Name string } -// TODO: Right now, there is only one resolver type. When more are added, this will need to be updated. -func NewResolver(imageReference string, serviceAccountName string) Resolver { - return OCIResolver{ - imageReference: imageReference, - keychainProvider: func() (authn.Keychain, error) { - return k8schain.NewInCluster(k8schain.Options{ServiceAccountName: serviceAccountName}) - }, - } +// Resolver defines a generic API to retrieve Tekton resources from remote locations. It allows 2 principle operations: +// - List: retrieve a flat set of Tekton objects in this remote location +// - Get: retrieves a specific object with the given Kind and name. +type Resolver interface { + List() ([]ResolvedObject, error) + Get(kind, name string) (runtime.Object, error) } diff --git a/test/builder/task.go b/test/builder/task.go index ab675f55031..4e6dc3cee8c 100644 --- a/test/builder/task.go +++ b/test/builder/task.go @@ -108,6 +108,16 @@ func Task(name string, ops ...TaskOp) *v1alpha1.Task { return t } +// TaskType sets the TypeMeta on the Task which is useful for making it serializable/deserializable. +func TaskType() TaskOp { + return func(t *v1alpha1.Task) { + t.TypeMeta = metav1.TypeMeta{ + APIVersion: "tekton.dev/v1alpha1", + Kind: "Task", + } + } +} + // ClusterTask creates a ClusterTask with default values. // Any number of ClusterTask modifier can be passed to transform it. func ClusterTask(name string, ops ...ClusterTaskOp) *v1alpha1.ClusterTask { @@ -131,6 +141,16 @@ func TaskNamespace(namespace string) TaskOp { } } +// ClusterTaskType sets the TypeMeta on the ClusterTask which is useful for making it serializable/deserializable. +func ClusterTaskType() ClusterTaskOp { + return func(t *v1alpha1.ClusterTask) { + t.TypeMeta = metav1.TypeMeta{ + APIVersion: "tekton.dev/v1alpha1", + Kind: "ClusterTask", + } + } +} + // ClusterTaskSpec sets the specified spec of the cluster task. // Any number of TaskSpec modifier can be passed to create it. func ClusterTaskSpec(ops ...TaskSpecOp) ClusterTaskOp { diff --git a/test/builder/task_test.go b/test/builder/task_test.go index 5821034913d..d0afb8268d1 100644 --- a/test/builder/task_test.go +++ b/test/builder/task_test.go @@ -42,7 +42,7 @@ var ( ) func TestTask(t *testing.T) { - task := tb.Task("test-task", tb.TaskSpec( + task := tb.Task("test-task", tb.TaskType(), tb.TaskSpec( tb.TaskInputs( tb.InputsResource("workspace", v1alpha1.PipelineResourceTypeGit, tb.ResourceTargetPath("/foo/bar")), tb.InputsResource("optional_workspace", v1alpha1.PipelineResourceTypeGit, tb.ResourceOptional(true)), @@ -67,6 +67,10 @@ func TestTask(t *testing.T) { tb.TaskWorkspace("bread", "kind of bread", "/bread/path", false), ), tb.TaskNamespace("foo")) expectedTask := &v1alpha1.Task{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "tekton.dev/v1alpha1", + Kind: "Task", + }, ObjectMeta: metav1.ObjectMeta{Name: "test-task", Namespace: "foo"}, Spec: v1alpha1.TaskSpec{ TaskSpec: v1beta1.TaskSpec{ @@ -143,12 +147,16 @@ func TestTask(t *testing.T) { } func TestClusterTask(t *testing.T) { - task := tb.ClusterTask("test-clustertask", tb.ClusterTaskSpec( + task := tb.ClusterTask("test-clustertask", tb.ClusterTaskType(), tb.ClusterTaskSpec( tb.Step("myimage", tb.StepCommand("/mycmd"), tb.StepArgs( "--my-other-arg=$(inputs.resources.workspace.url)", )), )) expectedTask := &v1alpha1.ClusterTask{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "tekton.dev/v1alpha1", + Kind: "ClusterTask", + }, ObjectMeta: metav1.ObjectMeta{Name: "test-clustertask"}, Spec: v1alpha1.TaskSpec{TaskSpec: v1beta1.TaskSpec{ Steps: []v1alpha1.Step{{Container: corev1.Container{ diff --git a/test/remote.go b/test/remote.go new file mode 100644 index 00000000000..cf94dc6ef03 --- /dev/null +++ b/test/remote.go @@ -0,0 +1,84 @@ +/* +Copyright 2020 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "bytes" + "fmt" + "reflect" + "strings" + + "github.com/ghodss/yaml" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + remoteimg "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "k8s.io/apimachinery/pkg/runtime" +) + +// CreateImage will push a new OCI image artifact with the provided raw data object as a layer and return the full image +// reference with a digest to fetch the image. Key must be specified as [lowercase kind]/[object name]. The image ref +// with a digest is returned. +func CreateImage(ref string, objs ...runtime.Object) (string, error) { + imgRef, err := name.ParseReference(ref) + if err != nil { + return "", fmt.Errorf("undexpected error producing image reference %w", err) + } + + img := empty.Image + + for _, obj := range objs { + data, err := yaml.Marshal(obj) + if err != nil { + return "", fmt.Errorf("error serializing object: %w", err) + } + + layer, err := tarball.LayerFromReader(bytes.NewReader(data)) + if err != nil { + return "", fmt.Errorf("unexpected error adding layer to image %w", err) + } + + img, err = mutate.Append(img, mutate.Addendum{ + Layer: layer, + Annotations: map[string]string{ + "org.opencontainers.image.title": getObjectName(obj), + "cdf.tekton.image.kind": strings.ToLower(obj.GetObjectKind().GroupVersionKind().Kind), + "cdf.tekton.image.apiVersion": strings.ToLower(obj.GetObjectKind().GroupVersionKind().Version), + }, + }) + if err != nil { + return "", fmt.Errorf("could not add layer to image %w", err) + } + } + + if err := remoteimg.Write(imgRef, img); err != nil { + return "", fmt.Errorf("could not push example image to registry") + } + + digest, err := img.Digest() + if err != nil { + return "", fmt.Errorf("could not read image digest: %w", err) + } + + return imgRef.Context().Digest(digest.String()).String(), nil +} + +// Return the ObjectMetadata.Name field which every resource should have. +func getObjectName(obj runtime.Object) string { + return reflect.Indirect(reflect.ValueOf(obj)).FieldByName("ObjectMeta").FieldByName("Name").String() +} diff --git a/test/remote_test.go b/test/remote_test.go new file mode 100644 index 00000000000..4fb3d0095f3 --- /dev/null +++ b/test/remote_test.go @@ -0,0 +1,96 @@ +/* +Copyright 2020 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "io/ioutil" + "net/http/httptest" + "net/url" + "testing" + + "github.com/ghodss/yaml" + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/remote" + tb "github.com/tektoncd/pipeline/test/builder" +) + +func TestCreateImage(t *testing.T) { + // Set up a fake registry to push an image to. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, _ := url.Parse(s.URL) + + task := tb.Task("test-create-image", tb.TaskType()) + + ref, err := CreateImage(u.Host+"/task/test-create-image", task) + if err != nil { + t.Errorf("uploading image failed unexpectedly with an error: %w", err) + } + + // Pull the image and ensure the layers are composed correctly. + imgRef, err := name.ParseReference(ref) + if err != nil { + t.Errorf("digest %s is not a valid reference: %w", ref, err) + } + + img, err := remote.Image(imgRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + if err != nil { + t.Errorf("could not fetch created image: %w", err) + } + + m, err := img.Manifest() + if err != nil { + t.Errorf("failed to fetch img manifest: %w", err) + } + + layers, err := img.Layers() + if err != nil || len(layers) != 1 { + t.Errorf("img layers were incorrect. Num Layers: %d. Err: %w", len(layers), err) + } + + if diff := cmp.Diff(m.Layers[0].Annotations["org.opencontainers.image.title"], "test-create-image"); diff != "" { + t.Error(diff) + } + if diff := cmp.Diff(m.Layers[0].Annotations["cdf.tekton.image.kind"], "task"); diff != "" { + t.Error(diff) + } + if diff := cmp.Diff(m.Layers[0].Annotations["cdf.tekton.image.apiVersion"], "v1alpha1"); diff != "" { + t.Error(diff) + } + + // Read the layer's contents and ensure it matches the task we uploaded. + rc, err := layers[0].Uncompressed() + if err != nil { + t.Errorf("layer contents were corrupted: %w", err) + } + defer rc.Close() + actual, err := ioutil.ReadAll(rc) + if err != nil { + t.Errorf("layer contents weren't readable: %w", err) + } + + raw, err := yaml.Marshal(task) + if err != nil { + t.Fatalf("Could not marshal task to bytes: %#v", err) + } + if diff := cmp.Diff(string(raw), string(actual)); diff != "" { + t.Error(diff) + } +} diff --git a/third_party/github.com/hashicorp/errwrap/go.mod b/third_party/github.com/hashicorp/errwrap/go.mod index c9b84022cf7..4bd9e8ea4bf 100644 --- a/third_party/github.com/hashicorp/errwrap/go.mod +++ b/third_party/github.com/hashicorp/errwrap/go.mod @@ -1 +1,3 @@ module github.com/hashicorp/errwrap + +go 1.14