From b64aca26266ee9258a47afda69373ea52f582254 Mon Sep 17 00:00:00 2001 From: numerology Date: Fri, 4 Oct 2019 09:09:30 -0700 Subject: [PATCH 1/4] Update README for MKP development. Remove managed SQL part and update verison tag. --- manifests/gcp_marketplace/README.md | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/manifests/gcp_marketplace/README.md b/manifests/gcp_marketplace/README.md index 61adef5e39c..17efe2de226 100644 --- a/manifests/gcp_marketplace/README.md +++ b/manifests/gcp_marketplace/README.md @@ -115,12 +115,6 @@ Creat the namespace kubectl create namespace $NAMESPACE ``` -Download token for your service account which you want to use for calling GCP APIs from the pipelines. -```shell -gcloud iam service-accounts keys create application_default_credentials.json --iam-account [your-service-account] -export SERVICE_ACCOUNT_TOKEN="$(cat application_default_credentials.json | base64 -w 0)" -``` - Follow the [instruction](https://github.com/GoogleCloudPlatform/marketplace-k8s-app-tools/blob/master/docs/tool-prerequisites.md#tool-prerequisites) and install mpdev TODO: The official mpdev won't work because it doesn't have permission to deploy CRD. The latest unofficial build will have right permission. Remove following instruction when change is in prod. ``` @@ -134,15 +128,10 @@ export MARKETPLACE_TOOLS_IMAGE=gcr.io/cloud-marketplace-staging/marketplace-k8s- Run the install script ```shell -mpdev scripts/install --deployer=gcr.io/ml-pipeline/google/pipelines/deployer:0.2 --parameters='{"name": "'$APP_INSTANCE_NAME'", "namespace": "'$NAMESPACE'", "serviceAccountCredential": "'$SERVICE_ACCOUNT_TOKEN'"}' +mpdev scripts/install --deployer=gcr.io/ml-pipeline/google/pipelines/deployer:0.1 --parameters='{"name": "'$APP_INSTANCE_NAME'", "namespace": "'$NAMESPACE'"}' ``` -Or if using CloudSQL and GCS, -``` -mpdev scripts/install --deployer=gcr.io/ml-pipeline/google/pipelines/deployer:0.2 --parameters='{"name": "'$APP_INSTANCE_NAME'", "namespace": "'$NAMESPACE'", "serviceAccountCredential": "'$SERVICE_ACCOUNT_TOKEN'", "managedstorage.enabled": true, "managedstorage.cloudsqlInstanceConnectionName": "[your-name]", "managedstorage.dbPassword": "[your-pwd]"}' -``` - Watch the deployment come up with ```shell From d3faa6fdfebf787efeb65226c6091037b2ff2b1d Mon Sep 17 00:00:00 2001 From: numerology Date: Fri, 4 Oct 2019 09:22:14 -0700 Subject: [PATCH 2/4] Update guide for MKP deployment. --- manifests/gcp_marketplace/guide.md | 40 +++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/manifests/gcp_marketplace/guide.md b/manifests/gcp_marketplace/guide.md index a6e11978aa1..4ed0eb9421c 100644 --- a/manifests/gcp_marketplace/guide.md +++ b/manifests/gcp_marketplace/guide.md @@ -17,26 +17,42 @@ Specify a [Kubenetes namespace](https://kubernetes.io/docs/concepts/overview/wor ## App instance name Specify an app instance name to help you identify this instance. +## Deploy +Click `Deploy` to start deploying Kubeflow Pipelines into the cluster you specified. +Deployment might take few minutes, so please be patient. After deployment is complete, go to the [Pipelines Console](http://pantheon.corp.google.com/ai-platform/pipelines) to access the Kubeflow Pipelines instance. + ## GCP Service Account credentials -This deployment requires a [GCP service account](https://cloud.google.com/iam/docs/service-accounts) to use for authentication when calling other GCP services. This includes Cloud Storage and Cloud SQL if you are using managed storage, as well as other services your pipeline might need, for example Dataflow. Specify the base64-encoded credentials for the service account you want to use. +After deployment, you can grant KFP proper permission by specifying its service account and binding +proper role to it. -You can get these credentials by running the following command in a terminal window. This command will create a new key under the service account. Please note that a single service account can only have 10 keys. +Usually a functional KFP pipeline requires a [GCP service account](https://cloud.google.com/iam/docs/service-accounts) to use for +authentication when calling other GCP services. This includes Cloud Storage as well as other services your pipeline might need, +for example Dataflow, Dataproc. Specify the base64-encoded credentials for the service account you want to use. +This can be done through command line using `kubectl`. ``` -$ gcloud iam service-accounts keys create application_default_credentials.json --iam-account [your-service-account] && cat application_default_credentials.json | base64 +export CLUSTER= +export ZONE= +# Configure kubectl to connect with the cluster +gcloud container clusters get-credentials "$CLUSTER" --zone "$ZONE" ``` - -Existing key also can be used. - +Then we can create and inject service account credential. ``` -cat existing_credentials.json | base64 +export PROJECT= +export SA_NAME= +# Create service account +gcloud iam service-accounts create $SA_NAME --display-name $SA_NAME +gcloud projects add-iam-policy-binding $PROJECT --member=serviceAccount:my-account@$PROJECT.iam.gserviceaccount.com --role=roles/storage.admin +# Also do this binding for other roles you need. For example, dataproc.admin and dataflow.admin +export SERVICE_ACCOUNT_TOKEN="$(cat application_default_credentials.json | base64 -w 0)" +echo -e "apiVersion: v1\nkind: Secret\nmetadata:\n name: \"user-gcp-sa\"\n namespace: \"${NAMESPACE}\"\n labels:\n app: gcp-sa\n app.kubernetes.io/name: \"${APP_INSTANCE_NAME}\"\ntype: Opaque\ndata:\n application_default_credentials.json: ${SERVICE_ACCOUNT_TOKEN}\n user-gcp-sa.json: $SERVICE_ACCOUNT_TOKEN" > secret.yaml +kubectl apply -f secret.yaml +# Remove secret files +rm application_default_credentials.json test.yaml ``` -If you are running this command on Linux, please use `base64 -w 0` to disable line wrapping. - -## Deploy -Click `Deploy` to start deploying Kubeflow Pipelines into the cluster you specified. -Deployment might take few minutes, so please be patient. After deployment is complete, go to the [Pipelines Console](http://pantheon.corp.google.com/ai-platform/pipelines) to access the Kubeflow Pipelines instance. +Note that the above commands use `base64 -w 0` to disable line wrapping, this could be slightly different +across platforms. ## Tips From 9ee75f573640fab97cbf2e4c99c9cfcf824a74eb Mon Sep 17 00:00:00 2001 From: numerology Date: Fri, 4 Oct 2019 09:54:50 -0700 Subject: [PATCH 3/4] Small fix. --- manifests/gcp_marketplace/guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/gcp_marketplace/guide.md b/manifests/gcp_marketplace/guide.md index 4ed0eb9421c..29a5d3c0f0d 100644 --- a/manifests/gcp_marketplace/guide.md +++ b/manifests/gcp_marketplace/guide.md @@ -36,7 +36,7 @@ export ZONE= # Configure kubectl to connect with the cluster gcloud container clusters get-credentials "$CLUSTER" --zone "$ZONE" ``` -Then we can create and inject service account credential. +Then you can create and inject service account credential. ``` export PROJECT= export SA_NAME= From 016b3ab15c7f84ecdbd542d482ab4a574bf81ce1 Mon Sep 17 00:00:00 2001 From: numerology Date: Fri, 4 Oct 2019 17:01:26 -0700 Subject: [PATCH 4/4] fix one missing thing --- manifests/gcp_marketplace/guide.md | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/gcp_marketplace/guide.md b/manifests/gcp_marketplace/guide.md index 29a5d3c0f0d..e342a9e2cff 100644 --- a/manifests/gcp_marketplace/guide.md +++ b/manifests/gcp_marketplace/guide.md @@ -44,6 +44,7 @@ export SA_NAME= gcloud iam service-accounts create $SA_NAME --display-name $SA_NAME gcloud projects add-iam-policy-binding $PROJECT --member=serviceAccount:my-account@$PROJECT.iam.gserviceaccount.com --role=roles/storage.admin # Also do this binding for other roles you need. For example, dataproc.admin and dataflow.admin +gcloud iam service-accounts keys create application_default_credentials.json --iam-account $SA_NAME@$PROJECT.iam.gserviceaccount.com export SERVICE_ACCOUNT_TOKEN="$(cat application_default_credentials.json | base64 -w 0)" echo -e "apiVersion: v1\nkind: Secret\nmetadata:\n name: \"user-gcp-sa\"\n namespace: \"${NAMESPACE}\"\n labels:\n app: gcp-sa\n app.kubernetes.io/name: \"${APP_INSTANCE_NAME}\"\ntype: Opaque\ndata:\n application_default_credentials.json: ${SERVICE_ACCOUNT_TOKEN}\n user-gcp-sa.json: $SERVICE_ACCOUNT_TOKEN" > secret.yaml kubectl apply -f secret.yaml