Add readme and update application parameter (#2021)

* pass pipeline runner service account to api server * update schema * fix * change image * rename * fix * fix * update readme * update readme
kubeflow · Sep 3, 2019 · 04ddb32 · 04ddb32
1 parent ecd93a5
commit 04ddb32
Show file tree

Hide file tree

Showing 10 changed files with 81 additions and 59 deletions.
diff --git a/manifests/gcp_marketplace/README.md b/manifests/gcp_marketplace/README.md
@@ -7,4 +7,41 @@ This directory contains the stacks to deploy on Google Cloud Marketplace.
 Popular open stacks on Kubernetes packaged by Google.
 
 # Installation
-TODO
+
+Build docker image
+```
+docker build --tag gcr.io/ml-pipeline/google/kfp/deployer:0.1.27 -f deployer/Dockerfile .
+docker push gcr.io/ml-pipeline/google/kfp/deployer:0.1.27
+```
+
+Install application CRD in a cluster
+```
+kubectl apply -f "https://raw.githubusercontent.com/GoogleCloudPlatform/marketplace-k8s-app-tools/master/crd/app-crd.yaml"
+```
+
+Install mpdev
+```
+BIN_FILE="$HOME/bin/mpdev"
+docker run gcr.io/cloud-marketplace-staging/marketplace-k8s-app-tools/k8s/dev:unreleased-pr396 cat /scripts/dev > "$BIN_FILE"
+chmod +x "$BIN_FILE"
+export MARKETPLACE_TOOLS_TAG=unreleased-pr396
+export MARKETPLACE_TOOLS_IMAGE=gcr.io/cloud-marketplace-staging/marketplace-k8s-app-tools/k8s/dev
+```
+
+Install Kubeflow Pipelines
+```
+kubectl create ns test
+mpdev /scripts/install --deployer=gcr.io/ml-pipeline/google/kfp/deployer:0.1.27 --parameters='{"name": "installation-1", "namespace": "test"}'
+```
+
+Install with Cloud SQL and GCS
+```
+mpdev /scripts/install \
+  --deployer=gcr.io/ml-pipeline/google/kfp/deployer:0.1.27 \
+  --parameters='{"name": "installation-1", "namespace": "test" , "managedstorage.enabled": true, "managedstorage.cloudsqlInstanceConnectionName": "your-instance-name", "managedstorage.dbPassword": "your password"}'
+```
+
+To uninstall 
+```
+kubectl delete applications -n test installation-1
+```
diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/argo.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/argo.yaml
@@ -19,7 +19,7 @@ data:
   config: |
     {
     namespace: {{ .Release.Namespace }},
-    executorImage: {{ .Values.images.argoExecutor }},
+    executorImage: {{ .Values.images.argoexecutor }},
     artifactRepository:
     {
         s3: {
@@ -87,7 +87,7 @@ spec:
             - --configmap
             - workflow-controller-configmap
             - --executor-image
-            - {{ .Values.images.argoExecutor }}
+            - {{ .Values.images.argoexecutor }}
           command:
             - workflow-controller
           env:
@@ -96,7 +96,7 @@ spec:
                 fieldRef:
                   apiVersion: v1
                   fieldPath: metadata.namespace
-          image: {{ .Values.images.argoWorkflowController }}
+          image: {{ .Values.images.argoworkflowcontroller }}
           imagePullPolicy: IfNotPresent
           name: workflow-controller
           resources: {}

diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/metadata.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/metadata.yaml
@@ -40,7 +40,7 @@ spec:
             - --mysql_service_host=mysql
             - --mysql_service_port=3306
             - --mlmd_db_name={{ .Values.managedstorage.databaseNamePrefix }}_metadata
-          image: {{ .Values.images.metadataServer }}
+          image: {{ .Values.images.metadataserver }}
           name: container
           ports:
             - containerPort: 8080

diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/mysql.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/mysql.yaml
@@ -36,19 +36,13 @@ spec:
         app.kubernetes.io/name: {{ .Release.Name }}
     spec:
       containers:
-        - image: {{ .Values.images.cloudsqlProxy}}
+        - image: {{ .Values.images.cloudsqlproxy}}
           name: cloudsqlproxy
           env:
-            - name: GCP_PROJECT
-              value: {{ .Values.managedstorage.gcpProject }}
-            - name: CLOUDSQL_ZONE
-              value: {{ .Values.managedstorage.cloudsqlZone }}
-            - name: CLOUDSQL_INSTANCE_NAME
-              value: {{ .Values.managedstorage.cloudsqlInstanceName }}
           command: ["/cloud_sql_proxy",
                     "-dir=/cloudsql",
             # Replace with your own CloudSQL instance ID
-                    "-instances=$(GCP_PROJECT):$(CLOUDSQL_ZONE):$(CLOUDSQL_INSTANCE_NAME)=tcp:0.0.0.0:3306",
+                    "-instances={{ .Values.managedstorage.cloudsqlInstanceConnectionName }}=tcp:0.0.0.0:3306",
                     "-credential_file=/credentials/application_default_credentials.json",
                     "term_timeout=10s"]
           # set term_timeout if require graceful handling of shutdown

diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/pipeline.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/pipeline.yaml
@@ -196,7 +196,7 @@ spec:
               value: "50"
             - name: NAMESPACE
               value: {{ .Release.Namespace }}
-          image: {{ .Values.images.viewerCrd }}
+          image: {{ .Values.images.viewercrd }}
           imagePullPolicy: Always
           name: ml-pipeline-viewer-crd
       serviceAccountName: {{ .Values.serviceAccount.mlPipelineViewerCrd}}
@@ -220,7 +220,7 @@ spec:
         app.kubernetes.io/name: {{ .Release.Name }}
     spec:
       containers:
-        - image: {{ .Values.images.visualizationServer }}
+        - image: {{ .Values.images.visualizationserver }}
           imagePullPolicy: IfNotPresent
           name: ml-pipeline-visualizationserver
           ports:
@@ -255,13 +255,13 @@ spec:
             # Following environment variables are only needed when using Cloud SQL and GCS.
             {{ if .Values.managedstorage.enabled }}
             - name: OBJECTSTORECONFIG_BUCKETNAME
-              value: "{{ .Values.managedstorage.gcpProject }}-{{ .Values.managedstorage.cloudsqlInstanceName }}-{{ .Values.managedstorage.cloudsqlZone }}-{{ .Values.managedstorage.databaseNamePrefix }}"
+              value: "{{ .Values.managedstorage.cloudsqlInstanceConnectionName | replace ':' '-' }}-{{ .Values.managedstorage.databaseNamePrefix }}"
             - name: DBCONFIG_DBNAME
               value: "{{ .Values.managedstorage.databaseNamePrefix }}_pipeline"
             - name: DBCONFIG_PASSWORD
               value: "{{ .Values.managedstorage.dbPassword }}"
             {{ end }}
-          image: {{ .Values.images.apiServer }}
+          image: {{ .Values.images.apiserver }}
           imagePullPolicy: IfNotPresent
           name: ml-pipeline-api-server
           ports:

diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/proxy.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/proxy.yaml
@@ -17,7 +17,7 @@ spec:
         app.kubernetes.io/name: {{ .Release.Name }}
     spec:
       containers:
-        - image: {{ .Values.images.proxyAgent }}
+        - image: {{ .Values.images.proxyagent }}
           imagePullPolicy: IfNotPresent
           name: proxy-agent
       serviceAccountName: {{ .Values.serviceAccount.proxyAgentRunner}}
diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/values.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/values.yaml
@@ -1,17 +1,17 @@
 images:
-  apiServer: gcr.io/ml-pipeline/api-server:0.1.27
-  argoExecutor: gcr.io/ml-pipeline/argoexec:v2.3.0
-  argoWorkflowController: gcr.io/ml-pipeline/workflow-controller:v2.3.0
-  cloudsqlProxy: gcr.io/cloudsql-docker/gce-proxy:1.14
+  apiserver: gcr.io/ml-pipeline/api-server:0.1.27
+  argoexecutor: gcr.io/ml-pipeline/argoexec:v2.3.0
+  argoworkflowcontroller: gcr.io/ml-pipeline/workflow-controller:v2.3.0
+  cloudsqlproxy: gcr.io/cloudsql-docker/gce-proxy:1.14
   frontend: gcr.io/ml-pipeline/frontend:0.1.27
-  metadataServer: gcr.io/kubeflow-images-public/metadata:v0.1.8
+  metadataserver: gcr.io/kubeflow-images-public/metadata:v0.1.8
   minio: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z
   mysql: gcr.io/ml-pipeline/mysql:5.6
   persistenceagent: gcr.io/ml-pipeline/persistenceagent:0.1.27
-  proxyAgent: gcr.io/ml-pipeline/inverse-proxy-agent:0.1.27
+  proxyagent: gcr.io/ml-pipeline/inverse-proxy-agent:0.1.27
   scheduledworkflow: gcr.io/ml-pipeline/scheduledworkflow:0.1.27
-  viewerCrd: gcr.io/ml-pipeline/viewer-crd-controller:0.1.27
-  visualizationServer: gcr.io/ml-pipeline/visualization-server:0.1.27
+  viewercrd: gcr.io/ml-pipeline/viewer-crd-controller:0.1.27
+  visualizationserver: gcr.io/ml-pipeline/visualization-server:0.1.27
 
 serviceAccount:
   argo: null
@@ -23,11 +23,10 @@ serviceAccount:
   pipelineRunner: null
   proxyAgentRunner: null
 
+gcpSecretName: null
+
 managedstorage:
-  cloudsqlInstanceName: null
-  cloudsqlZone: null
+  enabled: false
+  cloudsqlInstanceConnectionName: null
   databaseNamePrefix: null
   dbPassword: null
-  enabled: false
-  gcpProject: null
-  gcpSecretName: null
diff --git a/manifests/gcp_marketplace/schema.yaml b/manifests/gcp_marketplace/schema.yaml
@@ -25,17 +25,17 @@ x-google-marketplace:
       properties:
         images.minio:
           type: FULL
-    argoExecutor:
+    argo-executor:
       properties:
-        images.argoExecutor:
+        images.argo-executor:
           type: FULL
-    argoWorkflowController:
+    argo-workflow-controller:
       properties:
-        images.argoWorkflowController:
+        images.argo-workflow-controller:
           type: FULL
-    apiServer:
+    api-server:
       properties:
-        images.apiServer:
+        images.api-server:
           type: FULL
     persistenceagent:
       properties:
@@ -49,25 +49,25 @@ x-google-marketplace:
       properties:
         images.frontend:
           type: FULL
-    viewerCrd:
+    viewer-crd:
       properties:
-        images.viewerCrd:
+        images.viewer-crd:
           type: FULL
-    visualizationServer:
+    visualization-server:
       properties:
-        images.visualizationServer:
+        images.visualization-server:
           type: FULL
-    metadataServer:
+    metadata-server:
       properties:
-        images.metadataServer:
+        images.metadata-server:
           type: FULL
-    proxyAgent:
+    proxy-agent:
       properties:
-        images.proxyAgent:
+        images.proxy-agent:
           type: FULL
-    cloudsqlProxy:
+    cloudsql-proxy:
       properties:
-        images.cloudsqlProxy:
+        images.cloudsql-proxy:
           type: FULL
   deployerServiceAccount:
     roles:

diff --git a/manifests/kustomize/env/gcp/gcp-configurations-patch.yaml b/manifests/kustomize/env/gcp/gcp-configurations-patch.yaml
@@ -25,10 +25,6 @@ spec:
       containers:
         - name: cloudsqlproxy
           env:
-            - name: GCP_PROJECT
-              value: ''
-            - name: CLOUDSQL_ZONE
-              # E.g. us-central1
-              value: ''
-            - name: CLOUDSQL_INSTANCE_NAME
+            - name: CLOUD_SQL_INSTANCE_CONNECTION_NAME
+              # E.g. project-id:us-central1:instance-name
               value: ''
diff --git a/manifests/kustomize/env/gcp/mysql/cloudsql-proxy-deployment.yaml b/manifests/kustomize/env/gcp/mysql/cloudsql-proxy-deployment.yaml
@@ -18,16 +18,12 @@ spec:
         - image: gcr.io/cloudsql-docker/gce-proxy:1.14
           name: cloudsqlproxy
           env:
-            - name: GCP_PROJECT
-              value: ""
-            - name: CLOUDSQL_ZONE
-              value: ""
-            - name: CLOUDSQL_INSTANCE_NAME
+            - name: CLOUD_SQL_INSTANCE_CONNECTION_NAME
               value: ""
           command: ["/cloud_sql_proxy",
                     "-dir=/cloudsql",
                     # Replace with your own CloudSQL instance ID
-                    "-instances=$(GCP_PROJECT):$(CLOUDSQL_ZONE):$(CLOUDSQL_INSTANCE_NAME)=tcp:0.0.0.0:3306",
+                    "-instances=$(CLOUD_SQL_INSTANCE_CONNECTION_NAME)=tcp:0.0.0.0:3306",
                     "-credential_file=/credentials/application_default_credentials.json",
                     "term_timeout=10s"]
           # set term_timeout if require graceful handling of shutdown