diff --git a/Makefile b/Makefile index 14dba26e885..00385855571 100644 --- a/Makefile +++ b/Makefile @@ -11,13 +11,14 @@ TEKTON_INSTALLS=./tekton/templates/installs hydrate: find $(REPO_DIRS)/kf-ci-v1/namespaces/auto-deploy -type f -not -name namespace.yaml -exec rm {} ";" rm -f $(REPO_DIRS)/kf-ci-v1/namespaces/auto-deploy/tekton* - rm -f $(REPO_DIRS)/kf-ci-v1/namespaces/kf-ci/kf-ci + rm -f $(REPO_DIRS)/kf-ci-v1/namespaces/kf-ci/kf-ci kustomize build -o $(REPO_DIRS)/kf-ci-v1/namespaces/auto-deploy $(TEKTON_INSTALLS)/auto-deploy kustomize build -o $(REPO_DIRS)/kf-ci-v1/namespaces/auto-deploy test-infra/auto-deploy/manifest kustomize build -o $(REPO_DIRS)/kf-ci-v1/namespaces/kf-ci $(TEKTON_INSTALLS)/kf-ci + cd test-infra/cleanup && make hydrate # Make sure there are no nomos errors -.PHONY: +.PHONY: acm-test: nomos vet --no-api-server-check --path=acm-repos/kf-ci-management nomos vet --no-api-server-check --path=acm-repos/kf-ci-v1 @@ -33,14 +34,14 @@ build-worker-image: set-worker-image: kpt cfg set ./tekton test-image $(shell yq r ./images/latest_image.json builds[0].tag) -update-worker-image: build-worker-image set-worker-image +update-worker-image: build-worker-image set-worker-image # This is a debug rule providing some sugar to hydrate and push the manifests and then wait for the # sync debug-push-and-run: make hydrate && git add . && git commit -m "Latest" && git push jlewi cd ./go/cmd/nomos-wait && go run . - kubectl --context=kf-ci-v1 create -f ./tekton/runs/nb-test-run.yaml + kubectl --context=kf-ci-v1 create -f ./tekton/runs/nb-test-run.yaml # This is a debug rule providing some sugar for fast iteration during development # It might need to be customized for your usage. @@ -52,4 +53,4 @@ debug-rebuild-and-run: make update-worker-image make hydrate && git add . && git commit -m "Latest" && git push jlewi cd ./go/cmd/nomos-wait && go run . - kubectl --context=kf-ci-v1 create -f ./tekton/runs/nb-test-run.yaml + kubectl --context=kf-ci-v1 create -f ./tekton/runs/nb-test-run.yaml diff --git a/acm-repos/kf-ci-v1/namespaces/auto-deploy/apps_v1_deployment_auto-deploy-server.yaml b/acm-repos/kf-ci-v1/namespaces/auto-deploy/apps_v1_deployment_auto-deploy-server.yaml index 3712f7e8f8d..facbb27c4a1 100644 --- a/acm-repos/kf-ci-v1/namespaces/auto-deploy/apps_v1_deployment_auto-deploy-server.yaml +++ b/acm-repos/kf-ci-v1/namespaces/auto-deploy/apps_v1_deployment_auto-deploy-server.yaml @@ -136,7 +136,7 @@ spec: serviceAccount: default-editor volumes: - configMap: - name: auto-deploy-config-2778d8c6c8 + name: auto-deploy-config-7f4chh5g64 name: config - emptyDir: {} name: cache diff --git a/acm-repos/kf-ci-v1/namespaces/auto-deploy/batch_v1beta1_cronjob_cleanup-ci-kubeflow-ci-deployment.yaml b/acm-repos/kf-ci-v1/namespaces/auto-deploy/batch_v1beta1_cronjob_cleanup-ci-kubeflow-ci-deployment.yaml index d0dc9da9d03..fe7e5da3d37 100644 --- a/acm-repos/kf-ci-v1/namespaces/auto-deploy/batch_v1beta1_cronjob_cleanup-ci-kubeflow-ci-deployment.yaml +++ b/acm-repos/kf-ci-v1/namespaces/auto-deploy/batch_v1beta1_cronjob_cleanup-ci-kubeflow-ci-deployment.yaml @@ -41,7 +41,7 @@ spec: serviceAccountName: default-editor volumes: - configMap: - name: cleanup-config-dh2846bbg5 + name: cleanup-config-556b6f2hdb name: cleanup-config schedule: 0 */2 * * * successfulJobsHistoryLimit: 3 diff --git a/acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml b/acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml index 913ac39f7de..1d3202bf7cd 100644 --- a/acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml +++ b/acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml @@ -57,6 +57,7 @@ spec: - auto-blueprints - --project=$(inputs.params.project) - --context=$(inputs.params.management-project) + - --dryrun=false env: - name: KUBECONFIG value: /workspace/kubeconfig diff --git a/acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_auto-deploy-config-2778d8c6c8.yaml b/acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_auto-deploy-config-7f4chh5g64.yaml similarity index 97% rename from acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_auto-deploy-config-2778d8c6c8.yaml rename to acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_auto-deploy-config-7f4chh5g64.yaml index 558216e9bc9..11a9bd386e5 100644 --- a/acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_auto-deploy-config-2778d8c6c8.yaml +++ b/acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_auto-deploy-config-7f4chh5g64.yaml @@ -132,12 +132,12 @@ data: \ type: git\n params:\n - name: revision\n value: v1.1-branch\n \ - name: url\n value: https://github.com/kubeflow/gcp-blueprints.git\n \ - name: testing-repo\n resourceSpec:\n type: git\n params:\n - - name: revision\n value: master\n - name: url\n value: + name: revision\n value: v1.1-branch\n - name: url\n value: https://github.com/kubeflow/testing.git\n # Need to use a KSA with appropriate - GSA\n serviceAccountName: default-editor\n pipelineRef:\n name: deploy-gcp-blueprint" + GSA\n serviceAccountName: default-editor\n pipelineRef:\n name: deploy-gcp-blueprint\n" kind: ConfigMap metadata: labels: service: auto-deploy - name: auto-deploy-config-2778d8c6c8 + name: auto-deploy-config-7f4chh5g64 namespace: auto-deploy diff --git a/acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_cleanup-config-dh2846bbg5.yaml b/acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_cleanup-config-556b6f2hdb.yaml similarity index 76% rename from acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_cleanup-config-dh2846bbg5.yaml rename to acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_cleanup-config-556b6f2hdb.yaml index 4b48b26d31e..556d06746af 100644 --- a/acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_cleanup-config-dh2846bbg5.yaml +++ b/acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_cleanup-config-556b6f2hdb.yaml @@ -4,9 +4,11 @@ data: up the Kubeflow auto-deployed blueprints.\n#\napiVersion: tekton.dev/v1alpha1\nkind: PipelineRun\nmetadata:\n generateName: cleanup-blueprints-\n namespace: auto-deploy\nspec:\n \ # TODO(jlewi): Override any parameters?\n #params: {}\n resources: \n - - name: testing-repo\n resourceSpec:\n type: git\n params:\n # - TODO(jlewi): Switch to master on kubeflow/gcp-blueprints\n - name: revision\n - \ value: master\n - name: url\n value: https://github.com/kubeflow/testing.git\n + name: testing-repo\n resourceSpec:\n type: git\n params:\n - + name: revision\n value: master\n - name: url\n value: + https://github.com/kubeflow/testing.git\n # Uncomment the following when + testing locally\n # - name: revision\n # value: cleanup-blueprints\n + \ # - name: url\n # value: https://github.com/Bobgy/testing.git\n \ # Need to use a KSA with appropriate GSA\n serviceAccountName: default-editor\n \ pipelineSpec:\n params:\n - name: management-cluster-name\n type: string\n description: The name of the management cluster. \n default: @@ -19,5 +21,5 @@ data: \ name: cleanup-kubeflow-ci\n kind: namespaced \n" kind: ConfigMap metadata: - name: cleanup-config-dh2846bbg5 + name: cleanup-config-556b6f2hdb namespace: auto-deploy diff --git a/acm-repos/kf-ci-v1/namespaces/kf-ci/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml b/acm-repos/kf-ci-v1/namespaces/kf-ci/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml index 177302ca81c..ac155e19156 100644 --- a/acm-repos/kf-ci-v1/namespaces/kf-ci/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml +++ b/acm-repos/kf-ci-v1/namespaces/kf-ci/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml @@ -57,6 +57,7 @@ spec: - auto-blueprints - --project=$(inputs.params.project) - --context=$(inputs.params.management-project) + - --dryrun=false env: - name: KUBECONFIG value: /workspace/kubeconfig diff --git a/py/kubeflow/testing/cleanup_blueprints.py b/py/kubeflow/testing/cleanup_blueprints.py index 7fef9e93841..83ae8de751a 100644 --- a/py/kubeflow/testing/cleanup_blueprints.py +++ b/py/kubeflow/testing/cleanup_blueprints.py @@ -95,13 +95,16 @@ def _delete_blueprints(namespace, to_keep_names, context=None, dryrun=True): for kind in kinds: client = cnrm_clients.CnrmClientApi(api_client, kind) - selector = "{0}=true".format(AUTO_DEPLOY_LABEL) + # https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#set-based-requirement + # selects resources with GROUP_LABEL set + selector = GROUP_LABEL results = client.list_namespaced(namespace, label_selector=selector) for i in results.get("items"): name = i["metadata"]["name"] + kf_name = i["metadata"].get("labels", {}).get(NAME_LABEL, "") - if name in to_keep_names: + if kf_name in to_keep_names: to_keep[kind].append(name) continue @@ -170,12 +173,12 @@ def auto_blueprints(project, context, dryrun=True, blueprints=None): # pylint: d # Use labels to identify auto-deployed instances auto_deploy_label = b["metadata"].get("labels", {}).get(AUTO_DEPLOY_LABEL, "false") - - is_auto_deploy = auto_deploy_label.lower() == "true" - - if not is_auto_deploy: - logging.info("Skipping cluster %s; its missing the auto-deploy label", - name) + blueprint_group = b["metadata"]["labels"].get(GROUP_LABEL, "unknown") + if blueprint_group == "unknown": + logging.info("Skipping cluster %s; its missing the %s label; it is not " + "an auto-deployed instance", + name, GROUP_LABEL) + continue # Tha name of blueprint kf_name = b["metadata"].get("labels", {}).get(NAME_LABEL, "") @@ -199,12 +202,6 @@ def auto_blueprints(project, context, dryrun=True, blueprints=None): # pylint: d logging.info("Blueprint %s is auto deployed", kf_name) - blueprint_group = b["metadata"]["labels"].get(GROUP_LABEL, "unknown") - - if blueprint_group == "unknown": - logging.warning("Blueprint %s was missing label %s", kf_name, - GROUP_LABEL) - if kf_name in auto_deployments[blueprint_group]: continue diff --git a/tekton/templates/tasks/cleanup-kubeflow-ci.yaml b/tekton/templates/tasks/cleanup-kubeflow-ci.yaml index 9e414091347..cc4cca77d9e 100644 --- a/tekton/templates/tasks/cleanup-kubeflow-ci.yaml +++ b/tekton/templates/tasks/cleanup-kubeflow-ci.yaml @@ -61,6 +61,7 @@ spec: - auto-blueprints - --project=$(inputs.params.project) - --context=$(inputs.params.management-project) + - --dryrun=false env: - name: KUBECONFIG value: /workspace/kubeconfig diff --git a/test-infra/cleanup/cleanup-blueprints-pipeline.yaml b/test-infra/cleanup/cleanup-blueprints-pipeline.yaml index 9c70368c3aa..50ca7d71d9d 100644 --- a/test-infra/cleanup/cleanup-blueprints-pipeline.yaml +++ b/test-infra/cleanup/cleanup-blueprints-pipeline.yaml @@ -14,11 +14,15 @@ spec: resourceSpec: type: git params: - # TODO(jlewi): Switch to master on kubeflow/gcp-blueprints - name: revision value: master - name: url value: https://github.com/kubeflow/testing.git + # Uncomment the following when testing locally + # - name: revision + # value: cleanup-blueprints + # - name: url + # value: https://github.com/Bobgy/testing.git # Need to use a KSA with appropriate GSA serviceAccountName: default-editor pipelineSpec: