From 19bc550a5e0b5a81641a9d898b7486294bf6a11a Mon Sep 17 00:00:00 2001 From: Dan Lorenc Date: Mon, 22 Jul 2019 13:22:36 -0500 Subject: [PATCH] Disallow the Cluster and Git resource types as Output resources. The cluster type currently has no defined behavior as output resources, and the git type behaves incorrectly. The new Volume type should be used instead for the use-cases supported by the old Git resource. This is part of the large cleanup in #1076, and should not be submitted until after #1087. --- pkg/apis/pipeline/v1alpha1/resource_types.go | 22 +- .../v1alpha1/taskrun/resources/apply_test.go | 6 +- .../taskrun/resources/input_resource_test.go | 101 +------ .../taskrun/resources/output_resource.go | 1 - .../taskrun/resources/output_resource_test.go | 246 +++++------------- pkg/reconciler/v1alpha1/taskrun/taskrun.go | 18 +- .../v1alpha1/taskrun/taskrun_test.go | 62 ++--- 7 files changed, 122 insertions(+), 334 deletions(-) diff --git a/pkg/apis/pipeline/v1alpha1/resource_types.go b/pkg/apis/pipeline/v1alpha1/resource_types.go index 15620a36db2..481bbc615ac 100644 --- a/pkg/apis/pipeline/v1alpha1/resource_types.go +++ b/pkg/apis/pipeline/v1alpha1/resource_types.go @@ -130,15 +130,29 @@ type PipelineResourceList struct { Items []PipelineResource `json:"items"` } -// ResourceFromType returns a PipelineResourceInterface from a PipelineResource's type. -func ResourceFromType(r *PipelineResource) (PipelineResourceInterface, error) { +// InputResourceFromType returns a PipelineResourceInterface from a PipelineResource's type. +func InputResourceFromType(r *PipelineResource) (PipelineResourceInterface, error) { switch r.Spec.Type { case PipelineResourceTypeGit: return NewGitResource(r) - case PipelineResourceTypeImage: - return NewImageResource(r) case PipelineResourceTypeCluster: return NewClusterResource(r) + default: + return InputOutputResourceFromType(r) + } +} + +// OutputResourceFromType returns a PipelineResourceInterface from a PipelineResource's type. +func OutputResourceFromType(r *PipelineResource) (PipelineResourceInterface, error) { + // We currently have no output-only resources. + return InputOutputResourceFromType(r) +} + +// InputOutputResourceFromType returns a PipelineResourceInterface from a PipelineResource's type. +func InputOutputResourceFromType(r *PipelineResource) (PipelineResourceInterface, error) { + switch r.Spec.Type { + case PipelineResourceTypeImage: + return NewImageResource(r) case PipelineResourceTypeStorage: return NewStorageResource(r) case PipelineResourceTypePullRequest: diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/apply_test.go b/pkg/reconciler/v1alpha1/taskrun/resources/apply_test.go index 920f6afc121..1586ad48f11 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/apply_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/apply_test.go @@ -160,7 +160,7 @@ var outputs = map[string]v1alpha1.PipelineResourceInterface{ "bucket": gcsResource, } -var gitResource, _ = v1alpha1.ResourceFromType(&v1alpha1.PipelineResource{ +var gitResource, _ = v1alpha1.InputResourceFromType(&v1alpha1.PipelineResource{ ObjectMeta: metav1.ObjectMeta{ Name: "git-resource", }, @@ -173,7 +173,7 @@ var gitResource, _ = v1alpha1.ResourceFromType(&v1alpha1.PipelineResource{ }, }) -var imageResource, _ = v1alpha1.ResourceFromType(&v1alpha1.PipelineResource{ +var imageResource, _ = v1alpha1.InputResourceFromType(&v1alpha1.PipelineResource{ ObjectMeta: metav1.ObjectMeta{ Name: "image-resource", }, @@ -186,7 +186,7 @@ var imageResource, _ = v1alpha1.ResourceFromType(&v1alpha1.PipelineResource{ }, }) -var gcsResource, _ = v1alpha1.ResourceFromType(&v1alpha1.PipelineResource{ +var gcsResource, _ = v1alpha1.InputResourceFromType(&v1alpha1.PipelineResource{ ObjectMeta: metav1.ObjectMeta{ Name: "gcs-resource", }, diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/input_resource_test.go b/pkg/reconciler/v1alpha1/taskrun/resources/input_resource_test.go index 714aaa20b3a..d35f9eca887 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/input_resource_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/input_resource_test.go @@ -189,7 +189,7 @@ func setUp(t *testing.T) { }} inputResourceInterfaces = make(map[string]v1alpha1.PipelineResourceInterface) for _, r := range rs { - ri, _ := v1alpha1.ResourceFromType(r) + ri, _ := v1alpha1.InputResourceFromType(r) inputResourceInterfaces[r.Name] = ri } } @@ -408,52 +408,6 @@ func TestAddResourceToTask(t *testing.T) { WorkingDir: "/workspace", }}, }, - }, { - desc: "git resource as input from previous task", - task: task, - taskRun: &v1alpha1.TaskRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: "get-from-git", - Namespace: "marshmallow", - OwnerReferences: []metav1.OwnerReference{{ - Kind: "PipelineRun", - Name: "pipelinerun", - }}, - }, - Spec: v1alpha1.TaskRunSpec{ - Inputs: v1alpha1.TaskRunInputs{ - Resources: []v1alpha1.TaskResourceBinding{{ - ResourceRef: v1alpha1.PipelineResourceRef{ - Name: "the-git", - }, - Name: "gitspace", - Paths: []string{"prev-task-path"}, - }}, - }, - }, - }, - wantErr: false, - want: &v1alpha1.TaskSpec{ - Inputs: gitInputs, - Steps: []corev1.Container{{ - Name: "create-dir-gitspace-mz4c7", - Image: "override-with-bash-noop:latest", - Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "mkdir -p /workspace/gitspace"}, - }, { - Name: "source-copy-gitspace-9l9zj", - Image: "override-with-bash-noop:latest", - Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "cp -r prev-task-path/. /workspace/gitspace"}, - VolumeMounts: []corev1.VolumeMount{{MountPath: "/pvc", Name: "pipelinerun-pvc"}}, - }}, - Volumes: []corev1.Volume{{ - Name: "pipelinerun-pvc", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "pipelinerun-pvc"}, - }, - }}, - }, }, { desc: "storage resource as input with target path", task: taskWithTargetPath, @@ -878,15 +832,6 @@ func TestStorageInputResource(t *testing.T) { } func TestAddStepsToTaskWithBucketFromConfigMap(t *testing.T) { - task := &v1alpha1.Task{ - ObjectMeta: metav1.ObjectMeta{ - Name: "build-from-repo", - Namespace: "marshmallow", - }, - Spec: v1alpha1.TaskSpec{ - Inputs: gitInputs, - }, - } taskWithTargetPath := &v1alpha1.Task{ ObjectMeta: metav1.ObjectMeta{ Name: "task-with-targetpath", @@ -903,44 +848,6 @@ func TestAddStepsToTaskWithBucketFromConfigMap(t *testing.T) { taskRun *v1alpha1.TaskRun want *v1alpha1.TaskSpec }{{ - desc: "git resource as input from previous task - copy to bucket", - task: task, - taskRun: &v1alpha1.TaskRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: "get-from-git", - Namespace: "marshmallow", - OwnerReferences: []metav1.OwnerReference{{ - Kind: "PipelineRun", - Name: "pipelinerun", - }}, - }, - Spec: v1alpha1.TaskRunSpec{ - Inputs: v1alpha1.TaskRunInputs{ - Resources: []v1alpha1.TaskResourceBinding{{ - ResourceRef: v1alpha1.PipelineResourceRef{ - Name: "the-git", - }, - Name: "gitspace", - Paths: []string{"prev-task-path"}, - }}, - }, - }, - }, - want: &v1alpha1.TaskSpec{ - Inputs: gitInputs, - Steps: []corev1.Container{{ - Name: "artifact-dest-mkdir-gitspace-mssqb", - Image: "override-with-bash-noop:latest", - Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "mkdir -p /workspace/gitspace"}, - }, { - Name: "artifact-copy-from-gitspace-78c5n", - Image: "override-with-gsutil-image:latest", - Command: []string{"/ko-app/gsutil"}, - Args: []string{"-args", "cp -P -r gs://fake-bucket/prev-task-path/* /workspace/gitspace"}, - }}, - }, - }, { desc: "storage resource as input from previous task - copy from bucket", task: taskWithTargetPath, taskRun: &v1alpha1.TaskRun{ @@ -967,12 +874,12 @@ func TestAddStepsToTaskWithBucketFromConfigMap(t *testing.T) { want: &v1alpha1.TaskSpec{ Inputs: gcsInputs, Steps: []corev1.Container{{ - Name: "artifact-dest-mkdir-workspace-6nl7g", + Name: "artifact-dest-mkdir-workspace-mssqb", Image: "override-with-bash-noop:latest", Command: []string{"/ko-app/bash"}, Args: []string{"-args", "mkdir -p /workspace/gcs-dir"}, }, { - Name: "artifact-copy-from-workspace-j2tds", + Name: "artifact-copy-from-workspace-78c5n", Image: "override-with-gsutil-image:latest", Command: []string{"/ko-app/gsutil"}, Args: []string{"-args", "cp -P -r gs://fake-bucket/prev-task-path/* /workspace/gcs-dir"}, @@ -1011,7 +918,7 @@ func mockResolveTaskResources(taskRun *v1alpha1.TaskRun) map[string]v1alpha1.Pip i = inputResourceInterfaces[name] resolved[r.Name] = i } else if r.ResourceSpec != nil { - i, _ = v1alpha1.ResourceFromType(&v1alpha1.PipelineResource{ + i, _ = v1alpha1.InputResourceFromType(&v1alpha1.PipelineResource{ ObjectMeta: metav1.ObjectMeta{ Name: r.Name, }, diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/output_resource.go b/pkg/reconciler/v1alpha1/taskrun/resources/output_resource.go index d5015276666..ba420be73d9 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/output_resource.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/output_resource.go @@ -35,7 +35,6 @@ var ( // an output that should be copied to the PVC allowedOutputResources = map[v1alpha1.PipelineResourceType]bool{ v1alpha1.PipelineResourceTypeStorage: true, - v1alpha1.PipelineResourceTypeGit: true, } ) diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/output_resource_test.go b/pkg/reconciler/v1alpha1/taskrun/resources/output_resource_test.go index ef628d361ce..a94a5b53622 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/output_resource_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/output_resource_test.go @@ -93,7 +93,7 @@ func outputResourceSetup(t *testing.T) { outputResources = make(map[string]v1alpha1.PipelineResourceInterface) for _, r := range rs { - ri, _ := v1alpha1.ResourceFromType(r) + ri, _ := v1alpha1.OutputResourceFromType(r) outputResources[r.Name] = ri } } @@ -107,134 +107,6 @@ func TestValidOutputResources(t *testing.T) { wantSteps []corev1.Container wantVolumes []corev1.Volume }{{ - name: "git resource in input and output", - desc: "git resource declared as both input and output with pipelinerun owner reference", - taskRun: &v1alpha1.TaskRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-taskrun-run-output-steps", - Namespace: "marshmallow", - OwnerReferences: []metav1.OwnerReference{{ - Kind: "PipelineRun", - Name: "pipelinerun", - }}, - }, - Spec: v1alpha1.TaskRunSpec{ - Inputs: v1alpha1.TaskRunInputs{ - Resources: []v1alpha1.TaskResourceBinding{{ - Name: "source-workspace", - ResourceRef: v1alpha1.PipelineResourceRef{ - Name: "source-git", - }, - }}, - }, - Outputs: v1alpha1.TaskRunOutputs{ - Resources: []v1alpha1.TaskResourceBinding{{ - Name: "source-workspace", - ResourceRef: v1alpha1.PipelineResourceRef{ - Name: "source-git", - }, - Paths: []string{"pipeline-task-name"}, - }}, - }, - }, - }, - task: &v1alpha1.Task{ - ObjectMeta: metav1.ObjectMeta{ - Name: "task1", - Namespace: "marshmallow", - }, - Spec: v1alpha1.TaskSpec{ - Inputs: &v1alpha1.Inputs{ - Resources: []v1alpha1.TaskResource{{ - Name: "source-workspace", - Type: "git", - }}, - }, - Outputs: &v1alpha1.Outputs{ - Resources: []v1alpha1.TaskResource{{ - Name: "source-workspace", - Type: "git", - }}, - }, - }, - }, - wantSteps: []corev1.Container{{ - Name: "source-mkdir-source-git-9l9zj", - Image: "override-with-bash-noop:latest", - Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "mkdir -p pipeline-task-name"}, - VolumeMounts: []corev1.VolumeMount{{ - Name: "pipelinerun-pvc", - MountPath: "/pvc", - }}, - }, { - Name: "source-copy-source-git-mz4c7", - Image: "override-with-bash-noop:latest", - Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "cp -r /workspace/source-workspace/. pipeline-task-name"}, - VolumeMounts: []corev1.VolumeMount{{ - Name: "pipelinerun-pvc", - MountPath: "/pvc", - }}, - }}, - }, { - name: "git resource in output only", - desc: "git resource declared as output with pipelinerun owner reference", - taskRun: &v1alpha1.TaskRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-taskrun-run-output-steps", - Namespace: "marshmallow", - OwnerReferences: []metav1.OwnerReference{{ - Kind: "PipelineRun", - Name: "pipelinerun", - }}, - }, - Spec: v1alpha1.TaskRunSpec{ - Outputs: v1alpha1.TaskRunOutputs{ - Resources: []v1alpha1.TaskResourceBinding{{ - Name: "source-workspace", - ResourceRef: v1alpha1.PipelineResourceRef{ - Name: "source-git", - }, - Paths: []string{"pipeline-task-name"}, - }}, - }, - }, - }, - task: &v1alpha1.Task{ - ObjectMeta: metav1.ObjectMeta{ - Name: "task1", - Namespace: "marshmallow", - }, - Spec: v1alpha1.TaskSpec{ - Outputs: &v1alpha1.Outputs{ - Resources: []v1alpha1.TaskResource{{ - Name: "source-workspace", - Type: "git", - }}, - }, - }, - }, - wantSteps: []corev1.Container{{ - Name: "source-mkdir-source-git-9l9zj", - Image: "override-with-bash-noop:latest", - Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "mkdir -p pipeline-task-name"}, - VolumeMounts: []corev1.VolumeMount{{ - Name: "pipelinerun-pvc", - MountPath: "/pvc", - }}, - }, { - Name: "source-copy-source-git-mz4c7", - Image: "override-with-bash-noop:latest", - Command: []string{"/ko-app/bash"}, - Args: []string{"-args", "cp -r /workspace/output/source-workspace/. pipeline-task-name"}, - VolumeMounts: []corev1.VolumeMount{{ - Name: "pipelinerun-pvc", - MountPath: "/pvc", - }}, - }}, - }, { name: "image resource in output with pipelinerun with owner", desc: "image resource declared as output with pipelinerun owner reference should not generate any steps", taskRun: &v1alpha1.TaskRun{ @@ -274,39 +146,6 @@ func TestValidOutputResources(t *testing.T) { }, wantSteps: nil, wantVolumes: nil, - }, { - name: "git resource in output", - desc: "git resource declared in output without pipelinerun owner reference", - taskRun: &v1alpha1.TaskRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-taskrun-run-output-steps", - Namespace: "marshmallow", - }, - Spec: v1alpha1.TaskRunSpec{ - Outputs: v1alpha1.TaskRunOutputs{ - Resources: []v1alpha1.TaskResourceBinding{{ - Name: "source-workspace", - ResourceRef: v1alpha1.PipelineResourceRef{ - Name: "source-git", - }, - }}, - }, - }, - }, - task: &v1alpha1.Task{ - ObjectMeta: metav1.ObjectMeta{ - Name: "task1", - Namespace: "marshmallow", - }, - Spec: v1alpha1.TaskSpec{ - Outputs: &v1alpha1.Outputs{ - Resources: []v1alpha1.TaskResource{{ - Name: "source-workspace", - Type: "git", - }}, - }, - }, - }, }, { name: "storage resource as both input and output", desc: "storage resource defined in both input and output with parents pipelinerun reference", @@ -716,8 +555,8 @@ func TestValidOutputResourcesWithBucketStorage(t *testing.T) { taskRun *v1alpha1.TaskRun wantSteps []corev1.Container }{{ - name: "git resource in input and output with bucket storage", - desc: "git resource declared as both input and output with pipelinerun owner reference", + name: "storage resource in input and output with bucket storage", + desc: "storage resource declared as both input and output with pipelinerun owner reference", taskRun: &v1alpha1.TaskRun{ ObjectMeta: metav1.ObjectMeta{ Name: "test-taskrun-run-output-steps", @@ -732,7 +571,7 @@ func TestValidOutputResourcesWithBucketStorage(t *testing.T) { Resources: []v1alpha1.TaskResourceBinding{{ Name: "source-workspace", ResourceRef: v1alpha1.PipelineResourceRef{ - Name: "source-git", + Name: "source-gcs", }, }}, }, @@ -740,7 +579,7 @@ func TestValidOutputResourcesWithBucketStorage(t *testing.T) { Resources: []v1alpha1.TaskResourceBinding{{ Name: "source-workspace", ResourceRef: v1alpha1.PipelineResourceRef{ - Name: "source-git", + Name: "source-gcs", }, Paths: []string{"pipeline-task-name"}, }}, @@ -756,26 +595,38 @@ func TestValidOutputResourcesWithBucketStorage(t *testing.T) { Inputs: &v1alpha1.Inputs{ Resources: []v1alpha1.TaskResource{{ Name: "source-workspace", - Type: "git", + Type: "storage", }}, }, Outputs: &v1alpha1.Outputs{ Resources: []v1alpha1.TaskResource{{ Name: "source-workspace", - Type: "git", + Type: "storage", }}, }, }, }, wantSteps: []corev1.Container{{ - Name: "artifact-copy-to-source-git-9l9zj", - Image: "override-with-gsutil-image:latest", - Command: []string{"/ko-app/gsutil"}, - Args: []string{"-args", "cp -P -r /workspace/source-workspace gs://fake-bucket/pipeline-task-name"}, - }}, + Name: "upload-source-gcs-9l9zj", + Image: "override-with-gsutil-image:latest", + Command: []string{"/ko-app/gsutil"}, + Args: []string{"-args", "rsync -d -r /workspace/source-workspace gs://some-bucket"}, + Env: []corev1.EnvVar{{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/var/secret/sname/key.json"}}, + VolumeMounts: []corev1.VolumeMount{{Name: "volume-source-gcs-sname", MountPath: "/var/secret/sname"}}, + }, + { + Name: "artifact-copy-to-source-gcs-mz4c7", + Image: "override-with-gsutil-image:latest", + Command: []string{"/ko-app/gsutil"}, + Args: []string{ + "-args", + "cp -P -r /workspace/source-workspace gs://fake-bucket/pipeline-task-name", + }, + }, + }, }, { - name: "git resource in output only with bucket storage", - desc: "git resource declared as output with pipelinerun owner reference", + name: "storage resource in output only with bucket storage", + desc: "storage resource declared as output with pipelinerun owner reference", taskRun: &v1alpha1.TaskRun{ ObjectMeta: metav1.ObjectMeta{ Name: "test-taskrun-run-output-steps", @@ -790,7 +641,7 @@ func TestValidOutputResourcesWithBucketStorage(t *testing.T) { Resources: []v1alpha1.TaskResourceBinding{{ Name: "source-workspace", ResourceRef: v1alpha1.PipelineResourceRef{ - Name: "source-git", + Name: "source-gcs", }, Paths: []string{"pipeline-task-name"}, }}, @@ -806,20 +657,32 @@ func TestValidOutputResourcesWithBucketStorage(t *testing.T) { Outputs: &v1alpha1.Outputs{ Resources: []v1alpha1.TaskResource{{ Name: "source-workspace", - Type: "git", + Type: "storage", }}, }, }, }, wantSteps: []corev1.Container{{ - Name: "artifact-copy-to-source-git-9l9zj", - Image: "override-with-gsutil-image:latest", - Command: []string{"/ko-app/gsutil"}, - Args: []string{"-args", "cp -P -r /workspace/output/source-workspace gs://fake-bucket/pipeline-task-name"}, - }}, + Name: "upload-source-gcs-9l9zj", + Image: "override-with-gsutil-image:latest", + Command: []string{"/ko-app/gsutil"}, + Args: []string{"-args", "rsync -d -r /workspace/output/source-workspace gs://some-bucket"}, + Env: []corev1.EnvVar{{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/var/secret/sname/key.json"}}, + VolumeMounts: []corev1.VolumeMount{{Name: "volume-source-gcs-sname", MountPath: "/var/secret/sname"}}, + }, + { + Name: "artifact-copy-to-source-gcs-mz4c7", + Image: "override-with-gsutil-image:latest", + Command: []string{"/ko-app/gsutil"}, + Args: []string{ + "-args", + "cp -P -r /workspace/output/source-workspace gs://fake-bucket/pipeline-task-name", + }, + }, + }, }, { - name: "git resource in output", - desc: "git resource declared in output without pipelinerun owner reference", + name: "storage resource in output", + desc: "storage resource declared in output without pipelinerun owner reference", taskRun: &v1alpha1.TaskRun{ ObjectMeta: metav1.ObjectMeta{ Name: "test-taskrun-run-output-steps", @@ -830,7 +693,7 @@ func TestValidOutputResourcesWithBucketStorage(t *testing.T) { Resources: []v1alpha1.TaskResourceBinding{{ Name: "source-workspace", ResourceRef: v1alpha1.PipelineResourceRef{ - Name: "source-git", + Name: "source-gcs", }, }}, }, @@ -845,11 +708,20 @@ func TestValidOutputResourcesWithBucketStorage(t *testing.T) { Outputs: &v1alpha1.Outputs{ Resources: []v1alpha1.TaskResource{{ Name: "source-workspace", - Type: "git", + Type: "storage", }}, }, }, }, + wantSteps: []corev1.Container{{ + Name: "upload-source-gcs-9l9zj", + Image: "override-with-gsutil-image:latest", + Command: []string{"/ko-app/gsutil"}, + Args: []string{"-args", "rsync -d -r /workspace/output/source-workspace gs://some-bucket"}, + Env: []corev1.EnvVar{{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/var/secret/sname/key.json"}}, + VolumeMounts: []corev1.VolumeMount{{Name: "volume-source-gcs-sname", MountPath: "/var/secret/sname"}}, + }, + }, }} { t.Run(c.name, func(t *testing.T) { outputResourceSetup(t) @@ -1022,7 +894,7 @@ func resolveOutputResources(taskRun *v1alpha1.TaskRun) map[string]v1alpha1.Pipel i = outputResources[name] resolved[r.Name] = i } else if r.ResourceSpec != nil { - i, _ = v1alpha1.ResourceFromType(&v1alpha1.PipelineResource{ + i, _ = v1alpha1.OutputResourceFromType(&v1alpha1.PipelineResource{ ObjectMeta: metav1.ObjectMeta{ Name: r.Name, }, diff --git a/pkg/reconciler/v1alpha1/taskrun/taskrun.go b/pkg/reconciler/v1alpha1/taskrun/taskrun.go index 08c1a032b63..cbccec2cbf8 100644 --- a/pkg/reconciler/v1alpha1/taskrun/taskrun.go +++ b/pkg/reconciler/v1alpha1/taskrun/taskrun.go @@ -384,12 +384,12 @@ func (c *Reconciler) updateReady(pod *corev1.Pod) error { // TODO(dibyom): Refactor resource setup/templating logic to its own function in the resources package func (c *Reconciler) createPod(tr *v1alpha1.TaskRun, rtr *resources.ResolvedTaskResources) (*corev1.Pod, error) { ts := rtr.TaskSpec.DeepCopy() - inputResources, err := resourceImplBinding(rtr.Inputs) + inputResources, err := inputResourceImplBinding(rtr.Inputs) if err != nil { c.Logger.Errorf("Failed to initialize input resources: %v", err) return nil, err } - outputResources, err := resourceImplBinding(rtr.Outputs) + outputResources, err := outputResourceImplBinding(rtr.Outputs) if err != nil { c.Logger.Errorf("Failed to initialize output resources: %v", err) return nil, err @@ -515,11 +515,13 @@ func isExceededResourceQuotaError(err error) bool { return err != nil && errors.IsForbidden(err) && strings.Contains(err.Error(), "exceeded quota") } +type resourceGetter func(r *v1alpha1.PipelineResource) (v1alpha1.PipelineResourceInterface, error) + // resourceImplBinding maps pipeline resource names to the actual resource type implementations -func resourceImplBinding(resources map[string]*v1alpha1.PipelineResource) (map[string]v1alpha1.PipelineResourceInterface, error) { +func resourceImplBinding(resources map[string]*v1alpha1.PipelineResource, getter resourceGetter) (map[string]v1alpha1.PipelineResourceInterface, error) { p := make(map[string]v1alpha1.PipelineResourceInterface) for rName, r := range resources { - i, err := v1alpha1.ResourceFromType(r) + i, err := getter(r) if err != nil { return nil, xerrors.Errorf("failed to create resource %s : %v with error: %w", rName, r, err) } @@ -527,3 +529,11 @@ func resourceImplBinding(resources map[string]*v1alpha1.PipelineResource) (map[s } return p, nil } + +func inputResourceImplBinding(resources map[string]*v1alpha1.PipelineResource) (map[string]v1alpha1.PipelineResourceInterface, error) { + return resourceImplBinding(resources, v1alpha1.InputResourceFromType) +} + +func outputResourceImplBinding(resources map[string]*v1alpha1.PipelineResource) (map[string]v1alpha1.PipelineResourceInterface, error) { + return resourceImplBinding(resources, v1alpha1.OutputResourceFromType) +} diff --git a/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go b/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go index eecf2003316..ad46c9b09d1 100644 --- a/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go @@ -91,7 +91,7 @@ var ( tb.InputsResource(gitResource.Name, v1alpha1.PipelineResourceTypeGit), tb.InputsResource(anotherGitResource.Name, v1alpha1.PipelineResourceTypeGit), ), - tb.TaskOutputs(tb.OutputsResource(gitResource.Name, v1alpha1.PipelineResourceTypeGit)), + tb.TaskOutputs(tb.OutputsResource(storageResource.Name, v1alpha1.PipelineResourceTypeStorage)), )) saTask = tb.Task("test-with-sa", "foo", tb.TaskSpec(tb.Step("sa-step", "foo", tb.Command("/mycmd")))) @@ -138,6 +138,10 @@ var ( gitResource = tb.PipelineResource("git-resource", "foo", tb.PipelineResourceSpec( v1alpha1.PipelineResourceTypeGit, tb.PipelineResourceSpecParam("URL", "https://foo.git"), )) + storageResource = tb.PipelineResource("storage-resource", "foo", tb.PipelineResourceSpec( + v1alpha1.PipelineResourceTypeStorage, tb.PipelineResourceSpecParam("Location", "gs://foo"), tb.PipelineResourceSpecParam("Type", "gcs"), + )) + anotherGitResource = tb.PipelineResource("another-git-resource", "foo", tb.PipelineResourceSpec( v1alpha1.PipelineResourceTypeGit, tb.PipelineResourceSpecParam("URL", "https://foobar.git"), )) @@ -266,7 +270,7 @@ func TestReconcile(t *testing.T) { ), ), tb.TaskRunOutputs( - tb.TaskRunOutputsResource(gitResource.Name, + tb.TaskRunOutputsResource(storageResource.Name, tb.TaskResourceBindingPaths("output-folder"), ), ), @@ -377,7 +381,7 @@ func TestReconcile(t *testing.T) { TaskRuns: taskruns, Tasks: []*v1alpha1.Task{simpleTask, saTask, templatedTask, outputTask, taskEnvTask}, ClusterTasks: []*v1alpha1.ClusterTask{clustertask}, - PipelineResources: []*v1alpha1.PipelineResource{gitResource, anotherGitResource, imageResource}, + PipelineResources: []*v1alpha1.PipelineResource{gitResource, anotherGitResource, imageResource, storageResource}, } for _, tc := range []struct { name string @@ -574,12 +578,12 @@ func TestReconcile(t *testing.T) { }, }, toolsVolume, downward, workspaceVolume, homeVolume), tb.PodRestartPolicy(corev1.RestartPolicyNever), - getCredentialsInitContainer("vr6ds"), + getCredentialsInitContainer("j2tds"), getPlaceToolsInitContainer(), - tb.PodContainer("step-create-dir-another-git-resource-78c5n", "override-with-bash-noop:latest", + tb.PodContainer("step-git-source-git-resource-9l9zj", "override-with-git:latest", tb.Command(entrypointLocation), - tb.Args("-wait_file", "/builder/downward/ready", "-post_file", "/builder/tools/0", "-wait_file_content", "-entrypoint", "/ko-app/bash", "--", - "-args", "mkdir -p /workspace/another-git-resource"), + tb.Args("-wait_file", "/builder/downward/ready", "-post_file", "/builder/tools/0", "-wait_file_content", "-entrypoint", "/ko-app/git-init", "--", + "-url", "https://foo.git", "-revision", "master", "-path", "/workspace/git-resource"), tb.WorkingDir(workspaceDir), tb.EnvVar("HOME", "/builder/home"), tb.VolumeMount("tools", "/builder/tools"), @@ -592,13 +596,12 @@ func TestReconcile(t *testing.T) { tb.EphemeralStorage("0"), )), ), - tb.PodContainer("step-source-copy-another-git-resource-mssqb", "override-with-bash-noop:latest", + tb.PodContainer("step-git-source-another-git-resource-mz4c7", "override-with-git:latest", tb.Command(entrypointLocation), - tb.Args("-wait_file", "/builder/tools/0", "-post_file", "/builder/tools/1", "-entrypoint", "/ko-app/bash", "--", - "-args", "cp -r source-folder/. /workspace/another-git-resource"), + tb.Args("-wait_file", "/builder/tools/0", "-post_file", "/builder/tools/1", "-entrypoint", "/ko-app/git-init", "--", + "-url", "https://foobar.git", "-revision", "master", "-path", "/workspace/another-git-resource"), tb.WorkingDir(workspaceDir), tb.EnvVar("HOME", "/builder/home"), - tb.VolumeMount("test-pvc", "/pvc"), tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), @@ -608,28 +611,11 @@ func TestReconcile(t *testing.T) { tb.EphemeralStorage("0"), )), ), - tb.PodContainer("step-create-dir-git-resource-mz4c7", "override-with-bash-noop:latest", - tb.Command(entrypointLocation), - tb.Args("-wait_file", "/builder/tools/1", "-post_file", "/builder/tools/2", "-entrypoint", "/ko-app/bash", "--", - "-args", "mkdir -p /workspace/git-resource"), - tb.WorkingDir(workspaceDir), - tb.EnvVar("HOME", "/builder/home"), - tb.VolumeMount("tools", "/builder/tools"), - tb.VolumeMount("workspace", workspaceDir), - tb.VolumeMount("home", "/builder/home"), - tb.Resources(tb.Requests( - tb.CPU("0"), - tb.Memory("0"), - tb.EphemeralStorage("0"), - )), - ), - tb.PodContainer("step-source-copy-git-resource-9l9zj", "override-with-bash-noop:latest", + tb.PodContainer("step-simple-step", "foo", tb.Command(entrypointLocation), - tb.Args("-wait_file", "/builder/tools/2", "-post_file", "/builder/tools/3", "-entrypoint", "/ko-app/bash", "--", - "-args", "cp -r source-folder/. /workspace/git-resource"), + tb.Args("-wait_file", "/builder/tools/1", "-post_file", "/builder/tools/2", "-entrypoint", "/mycmd", "--"), tb.WorkingDir(workspaceDir), tb.EnvVar("HOME", "/builder/home"), - tb.VolumeMount("test-pvc", "/pvc"), tb.VolumeMount("tools", "/builder/tools"), tb.VolumeMount("workspace", workspaceDir), tb.VolumeMount("home", "/builder/home"), @@ -639,9 +625,10 @@ func TestReconcile(t *testing.T) { tb.EphemeralStorage("0"), )), ), - tb.PodContainer("step-simple-step", "foo", + tb.PodContainer("step-upload-storage-resource-mssqb", "override-with-gsutil-image:latest", tb.Command(entrypointLocation), - tb.Args("-wait_file", "/builder/tools/3", "-post_file", "/builder/tools/4", "-entrypoint", "/mycmd", "--"), + tb.Args("-wait_file", "/builder/tools/2", "-post_file", "/builder/tools/3", "-entrypoint", "/ko-app/gsutil", "--", + "-args", "cp /workspace/output/storage-resource/* gs://foo"), tb.WorkingDir(workspaceDir), tb.EnvVar("HOME", "/builder/home"), tb.VolumeMount("tools", "/builder/tools"), @@ -653,10 +640,9 @@ func TestReconcile(t *testing.T) { tb.EphemeralStorage("0"), )), ), - tb.PodContainer("step-source-mkdir-git-resource-6nl7g", "override-with-bash-noop:latest", + tb.PodContainer("step-source-mkdir-storage-resource-78c5n", "override-with-bash-noop:latest", tb.Command(entrypointLocation), - tb.Args("-wait_file", "/builder/tools/4", "-post_file", "/builder/tools/5", "-entrypoint", "/ko-app/bash", "--", - "-args", "mkdir -p output-folder"), + tb.Args("-wait_file", "/builder/tools/3", "-post_file", "/builder/tools/4", "-entrypoint", "/ko-app/bash", "--", "-args", "mkdir -p output-folder"), tb.WorkingDir(workspaceDir), tb.EnvVar("HOME", "/builder/home"), tb.VolumeMount("test-pvc", "/pvc"), @@ -669,10 +655,10 @@ func TestReconcile(t *testing.T) { tb.EphemeralStorage("0"), )), ), - tb.PodContainer("step-source-copy-git-resource-j2tds", "override-with-bash-noop:latest", + tb.PodContainer("step-source-copy-storage-resource-6nl7g", "override-with-bash-noop:latest", tb.Command(entrypointLocation), - tb.Args("-wait_file", "/builder/tools/5", "-post_file", "/builder/tools/6", "-entrypoint", "/ko-app/bash", "--", - "-args", "cp -r /workspace/git-resource/. output-folder"), + tb.Args("-wait_file", "/builder/tools/4", "-post_file", "/builder/tools/5", "-entrypoint", "/ko-app/bash", "--", + "-args", "cp -r /workspace/output/storage-resource/. output-folder"), tb.WorkingDir(workspaceDir), tb.EnvVar("HOME", "/builder/home"), tb.VolumeMount("test-pvc", "/pvc"),