[Manifest] Cache - Enable cache and cache deployer in base kustomizat…

…ion file (kubeflow#3376) * Initial execution cache This commit adds initial execution cache service. Including http service and execution key generation. * fix master * Change cache deployer job to stateful set * Delete cache deployer job * Delete cache deployer job after it completes * minor fix * fix indention * Change cache deployer job to statefulset * Remove extra cluster role for cache deployer * remove cache in base kustomize file for upgrade test * minor fix * Enable cache and cache-deployer in base kustomization file * fix * fix * test * test * test * Refactor cluster scope resources * refactor * Add namespace for sa * Fix * Add crds folder to cluster kustomization yaml * namespace change * fix * fix * fix * update test * Rename cluster to cluster-scoped-resource * test adding namespace in kustomization file * revert namespace for clusterrolebinding * fix * Add db_name in cache_deployment manifest * rename * change secret cluster role to role
chensun · Apr 3, 2020 · 1faf721 · 1faf721
1 parent 44eae7e
commit 1faf721
Show file tree

Hide file tree

Showing 13 changed files with 45 additions and 23 deletions.
diff --git a/manifests/kustomize/README.md b/manifests/kustomize/README.md
@@ -10,7 +10,7 @@ Deploy latest version of Kubeflow Pipelines
 
 ```
 export PIPELINE_VERSION=0.3.0
-kubectl apply -f https://storage.googleapis.com/ml-pipeline/pipeline-lite/$PIPELINE_VERSION/crd.yaml
+kubectl apply -f https://storage.googleapis.com/ml-pipeline/pipeline-lite/$PIPELINE_VERSION/cluster-scoped-resources.yaml
 kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io
 kubectl apply -f https://storage.googleapis.com/ml-pipeline/pipeline-lite/$PIPELINE_VERSION/namespaced-install.yaml
 ```
@@ -44,7 +44,7 @@ To deploy Kubeflow Pipelines in namespace FOO,
 -   Then run
 
 ```
-kubectl kustomize base/crds | kubectl apply -f -
+kubectl kustomize cluster-scoped-resources | kubectl apply -f -
 # then
 kubectl kustomize env/dev | kubectl apply -f -
 # or
@@ -87,7 +87,7 @@ You can uninstall Kubeflow Pipelines by running
 ```
 export PIPELINE_VERSION=0.1.38
 kubectl delete -f https://storage.googleapis.com/ml-pipeline/pipeline-lite/$PIPELINE_VERSION/namespaced-install.yaml
-kubectl delete -f https://storage.googleapis.com/ml-pipeline/pipeline-lite/$PIPELINE_VERSION/crd.yaml
+kubectl delete -f https://storage.googleapis.com/ml-pipeline/pipeline-lite/$PIPELINE_VERSION/cluster-scoped-resources.yaml
 ```
 
 Or if you deploy through kustomize
@@ -97,7 +97,7 @@ kubectl kustomize env/dev | kubectl delete -f -
 # or
 kubectl kustomize env/gcp | kubectl delete -f -
 # then
-kubectl kustomize base/crds | kubectl delete -f -
+kubectl kustomize cluster-scoped-resources | kubectl delete -f -
 ```
 
 ## Upgrade

diff --git a/manifests/kustomize/base/cache-deployer/cache-deployer-rolebinding.yaml b/manifests/kustomize/base/cache-deployer/cache-deployer-rolebinding.yaml
@@ -4,8 +4,8 @@ metadata:
   name: kubeflow-pipelines-cache-deployer-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kubeflow-pipelines-cache-deployer-secret-clusterrole
+  kind: Role
+  name: kubeflow-pipelines-cache-deployer-secret-role
 subjects:
 - kind: ServiceAccount
   name: kubeflow-pipelines-cache-deployer-sa
diff --git a/...er/cache-deployer-secret-clusterrole.yaml → ...-deployer/cache-deployer-secret-role.yaml b/...er/cache-deployer-secret-clusterrole.yaml → ...-deployer/cache-deployer-secret-role.yaml
@@ -1,9 +1,9 @@
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
 metadata:
   labels:
-    app: kubeflow-pipelines-cache-deployer-secret-clusterrole
-  name: kubeflow-pipelines-cache-deployer-secret-clusterrole
+    app: kubeflow-pipelines-cache-deployer-secret-role
+  name: kubeflow-pipelines-cache-deployer-secret-role
 rules:
 - apiGroups:
   - ""

diff --git a/...-deployer/cache-deployer-clusterrole.yaml → ...r/cluster/cache-deployer-clusterrole.yaml b/...-deployer/cache-deployer-clusterrole.yaml → ...r/cluster/cache-deployer-clusterrole.yaml
diff --git a/...er/cache-deployer-clusterrolebinding.yaml → ...er/cache-deployer-clusterrolebinding.yaml b/...er/cache-deployer-clusterrolebinding.yaml → ...er/cache-deployer-clusterrolebinding.yaml
@@ -8,4 +8,5 @@ roleRef:
   name: kubeflow-pipelines-cache-deployer-clusterrole
 subjects:
 - kind: ServiceAccount
-  name: kubeflow-pipelines-cache-deployer-sa
+  name: kubeflow-pipelines-cache-deployer-sa
+  namespace: kubeflow
diff --git a/manifests/kustomize/base/cache-deployer/cluster/kustomization.yaml b/manifests/kustomize/base/cache-deployer/cluster/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - cache-deployer-clusterrole.yaml
+  - cache-deployer-clusterrolebinding.yaml
+
diff --git a/manifests/kustomize/base/cache-deployer/kustomization.yaml b/manifests/kustomize/base/cache-deployer/kustomization.yaml
@@ -2,9 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-  - cache-deployer-clusterrole.yaml
-  - cache-deployer-clusterrolebinding.yaml
-  - cache-deployer-secret-clusterrole.yaml
+  - cache-deployer-secret-role.yaml
   - cache-deployer-rolebinding.yaml
   - cache-deployer-sa.yaml
   - cache-deployer-statefulset.yaml

diff --git a/manifests/kustomize/base/cache/cache-deployment.yaml b/manifests/kustomize/base/cache/cache-deployment.yaml
@@ -45,6 +45,7 @@ spec:
         args: ["--db_driver=$(DBCONFIG_DRIVER)",
                "--db_host=$(DBCONFIG_HOST_NAME)",
                "--db_port=$(DBCONFIG_PORT)",
+               "--db_name=$(DBCONFIG_DB_NAME)",
                "--namespace_to_watch=$(NAMESPACE_TO_WATCH)",
               ]
         imagePullPolicy: Always

diff --git a/manifests/kustomize/base/kustomization.yaml b/manifests/kustomize/base/kustomization.yaml
@@ -7,6 +7,8 @@ bases:
 - pipeline
 - proxy
 - metadata
+- cache
+- cache-deployer
 
 resources:
   - pipeline-application.yaml
@@ -28,3 +30,7 @@ images:
   newTag: 0.3.0
 - name: gcr.io/ml-pipeline/metadata-writer
   newTag: 0.3.0
+- name: gcr.io/ml-pipeline/cache-server
+  newTag: 0.3.0
+- name: gcr.io/ml-pipeline/cache-deployer
+  newTag: 0.3.0
diff --git a/manifests/kustomize/cluster-scoped-resources/kustomization.yaml b/manifests/kustomize/cluster-scoped-resources/kustomization.yaml
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+- ../base/cache-deployer/cluster
+- ../base/crds
diff --git a/test/deploy-pipeline-lite.sh b/test/deploy-pipeline-lite.sh
@@ -41,7 +41,7 @@ if [ -z "$KFP_DEPLOY_RELEASE" ]; then
   echo "Deploying KFP in working directory..."
   KFP_MANIFEST_DIR=${DIR}/manifests
 
-  pushd ${KFP_MANIFEST_DIR}/crd
+  pushd ${KFP_MANIFEST_DIR}/cluster-scoped-resources
   kustomize build . | kubectl apply -f -
   kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io
   popd
@@ -58,8 +58,8 @@ if [ -z "$KFP_DEPLOY_RELEASE" ]; then
   kustomize edit set image gcr.io/ml-pipeline/visualization-server=${GCR_IMAGE_BASE_DIR}/visualization-server:${GCR_IMAGE_TAG}
   kustomize edit set image gcr.io/ml-pipeline/inverse-proxy-agent=${GCR_IMAGE_BASE_DIR}/inverse-proxy-agent:${GCR_IMAGE_TAG}
   kustomize edit set image gcr.io/ml-pipeline/metadata-writer=${GCR_IMAGE_BASE_DIR}/metadata-writer:${GCR_IMAGE_TAG}
-  kustomize edit set image gcr.io/ml-pipeline-test/cache-server=${GCR_IMAGE_BASE_DIR}/cache-server:${GCR_IMAGE_TAG}
-  kustomize edit set image gcr.io/ml-pipeline-test/cache-deployer=${GCR_IMAGE_BASE_DIR}/cache-deployer:${GCR_IMAGE_TAG}
+  kustomize edit set image gcr.io/ml-pipeline/cache-server=${GCR_IMAGE_BASE_DIR}/cache-server:${GCR_IMAGE_TAG}
+  kustomize edit set image gcr.io/ml-pipeline/cache-deployer=${GCR_IMAGE_BASE_DIR}/cache-deployer:${GCR_IMAGE_TAG}
   cat kustomization.yaml
 
   kustomize build . | kubectl apply -f -
@@ -71,13 +71,16 @@ else
   # temporarily checkout last release tag
   git checkout $KFP_LATEST_RELEASE
 
+  # TODO: rename crd to cluster-scoped-resources after the release next to 0.3.0
   pushd ${KFP_MANIFEST_DIR}/crd
   kustomize build . | kubectl apply -f -
   kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io
   popd
 
   pushd ${KFP_MANIFEST_DIR}/dev
   kustomize build . | kubectl apply -f -
+  # TODO: remove line below after the release next to 0.3.0
+  kubectl delete deployment cache-server -n ${NAMESPACE}
   popd
 
   # go back to previous commit
@@ -89,12 +92,17 @@ echo "Status of pods after kubectl apply"
 kubectl get pods -n ${NAMESPACE}
 
 # wait for all deployments to be successful
-# note, after we introduce statefulsets or daemonsets, we need to wait their rollout status here too
+# note, after we introduce daemonsets, we need to wait their rollout status here too
 for deployment in $(kubectl get deployments -n ${NAMESPACE} -o name)
 do
   kubectl rollout status $deployment -n ${NAMESPACE}
 done
 
+for statefulset in $(kubectl get statefulset -n ${NAMESPACE} -o name)
+do
+  kubectl rollout status $statefulset -n ${NAMESPACE}
+done
+
 echo "Status of pods after rollouts are successful"
 kubectl get pods -n ${NAMESPACE}
 

diff --git a/test/manifests/crd/kustomization.yaml → ...uster-scoped-resources/kustomization.yaml b/test/manifests/crd/kustomization.yaml → ...uster-scoped-resources/kustomization.yaml
@@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- ../../../manifests/kustomize/base/crds
+- ../../../manifests/kustomize/cluster-scoped-resources
diff --git a/test/manifests/dev/kustomization.yaml b/test/manifests/dev/kustomization.yaml
@@ -1,12 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-namespace: kubeflow
-
 # Actual image overrides will be added in test scripts.
 images: []
 resources:
 - ../../../manifests/kustomize/env/dev
-# Disables cache and cache-deployer temporarily because they block upgrade tests
-# - ../../../manifests/kustomize/base/cache
-# - ../../../manifests/kustomize/base/cache-deployer