Added manifest for deploying on aws using s3

manifests/kustomize/env/aws/README.md

@@ -0,0 +1,66 @@
+# Installing kubeflow pipelines for AWS infrastructure
+
+This is a guide on installing kubeflow pipelines in a kubernetes cluster
+(need not be EKS) hosted on AWS infrastructure (i.e. using EBS, S3, ...).
+
+There are 2 approaches towards allowing kubeflow pipelines services to access
+AWS resources:
+
+- AWS access keys (see [here](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys)), or
+- IAM (e.g. [kube2iam](https://github.com/jtblin/kube2iam))
+
+The kustomize overlays are available in the respective folders:
+
+- [accesskey](./accesskey): i.e. `kubectl kustomize accesskey`
+- [iam](./iam): i.e. `kubectl kustomize iam`
+
+## Quick start
+
+`params.env` in the overlay folders are the parameters that you can set for your
+deployment. Most of the defaults for parameters are fine except for:
+- `awsIAMRole`: for IAM-based approach, you need to provide the IAM role for the service to assume
+- `awsRegion`: your AWS region (this is needed for tensorboard viewer to work properly)
+
+```bash
+# generate the provided overlay variant
+kubectl kustomize [overlay_folder] > kubeflow-pipelines-aws.yaml
+# deploy
+kubectl apply -f kubeflow-pipelines-aws.yaml
+```
+
+## Notable changes
+
+> #### NOTE:
+>
+> The folder (defaults to `pipelines`) to save the pipeline templates cannot be
+> configured until
+> [#2080](https://github.com/kubeflow/pipelines/pull/2080) is merged in.
+
+#### Archiving pod logs
+`archiveLogs` is set to `true` so that pod logs are automatically archived into
+the configured S3 bucket. `ml-pipeline-ui` will be able to retrieve the pod logs
+even if the pod (and node) had been removed and purged.
+
+#### MySQL
+For simplicity, a generic mysql 5.6 service is provisioned. This can be replaced
+with AWS Aurora or MYSQL RDS if needed (but not tested).
+
+#### Access-key based access
+A k8s secret with the AWS credential must be created. This secret will be referenced
+by the various kfp services to access the S3 buckets.
+
+```bash
+kubectl -n kubeflow create secret generic ml-pipeline-aws-secret \
+    --from-literal=accesskey=$AWS_ACCESS_KEY_ID \
+    --from-literal=secretkey=$AWS_SECRET_ACCESS_KEY
+```
+
+#### IAM based access
+An appropriate IAM role must be created for kfp services to access the S3 buckets.
+
+This approach assumes that an IAM credential provisioning service
+(e.g. [kube2iam](https://github.com/jtblin/kube2iam)) is deployed in the k8s cluster.
+
+Alternatively, existing IAM role that is assigned to the k8s nodes can be updated to
+permit access to the appropriate S3 buckets (not recommended).
+

manifests/kustomize/env/aws/accesskey/aws-configurations-patch.yaml

@@ -0,0 +1,93 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: workflow-controller-configmap
+  namespace: kubeflow
+data:
+  config: |
+    {
+    executorImage: $(executorImage),
+    artifactRepository:
+    {
+        archiveLogs: true,
+        s3: {
+            bucket: $(artifactRepositoryBucket),
+            keyPrefix: $(artifactRepositoryKeyPrefix),
+            endpoint: s3.amazonaws.com,
+            insecure: false,
+            accessKeySecret: {
+                name: $(awsSecretName),
+                key: $(awsAccessKeySecretKey)
+            },
+            secretKeySecret: {
+                name: $(awsSecretName),
+                key: $(awsSecretKeySecretKey)
+            }
+        }
+    }
+    }
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: ml-pipeline-ui
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-ui
+          env:
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: $(awsSecretName)
+                  key: $(awsAccessKeySecretKey)
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: $(awsSecretName)
+                  key: $(awsSecretKeySecretKey)
+            - name: ARGO_ARCHIVE_LOGS
+              value: "true"
+            - name: ARGO_ARCHIVE_ARTIFACTORY
+              value: s3
+            - name: ARGO_ARCHIVE_BUCKETNAME
+              value: $(artifactRepositoryBucket)
+            - name: ARGO_ARCHIVE_PREFIX
+              value: $(artifactRepositoryKeyPrefix)
+            - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH
+              value: /etc/config/viewer-tensorboard-template.json
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/config
+      volumes:
+        - name: config-volume
+          configMap:
+            name: ml-pipeline-ui-viewer-template
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-api-server
+          env:
+            - name: OBJECTSTORECONFIG_ACCESSKEY
+              valueFrom:
+                secretKeyRef:
+                  name: $(awsSecretName)
+                  key: $(awsAccessKeySecretKey)
+            - name: OBJECTSTORECONFIG_SECRETACCESSKEY
+              valueFrom:
+                secretKeyRef:
+                  name: $(awsSecretName)
+                  key: $(awsSecretKeySecretKey)
+            - name: OBJECTSTORECONFIG_BUCKETNAME
+              value: $(artifactRepositoryBucket)
+            - name: MINIO_SERVICE_SERVICE_HOST
+              value: s3.amazonaws.com
+            - name: MINIO_SERVICE_SERVICE_PORT
+              value: "443"

manifests/kustomize/env/aws/accesskey/kustomization.yaml

@@ -0,0 +1,76 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+  - ../../../namespaced
+  - ../mysql
+
+resources:
+  - viewer-tensorboard-template-configmap.yaml
+
+# Replace with your namespace
+namespace: kubeflow
+
+patchesStrategicMerge:
+  - aws-configurations-patch.yaml
+
+configMapGenerator:
+- name: pipeline-aws-parameters
+  env: params.env
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+vars:
+- name: awsSecretName
+  objref:
+    kind: ConfigMap
+    name: pipeline-aws-parameters
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.awsSecretName
+- name: awsAccessKeySecretKey
+  objref:
+    kind: ConfigMap
+    name: pipeline-aws-parameters
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.awsAccessKeySecretKey
+- name: awsSecretKeySecretKey
+  objref:
+    kind: ConfigMap
+    name: pipeline-aws-parameters
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.awsSecretKeySecretKey
+- name: awsRegion
+  objref:
+    kind: ConfigMap
+    name: pipeline-aws-parameters
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.awsRegion
+- name: executorImage
+  objref:
+    kind: ConfigMap
+    name: pipeline-aws-parameters
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.executorImage
+- name: artifactRepositoryBucket
+  objref:
+    kind: ConfigMap
+    name: pipeline-aws-parameters
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.artifactRepositoryBucket
+- name: artifactRepositoryKeyPrefix
+  objref:
+    kind: ConfigMap
+    name: pipeline-aws-parameters
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.artifactRepositoryKeyPrefix
+
+configurations:
+- params.yaml

manifests/kustomize/env/aws/accesskey/params.env

@@ -0,0 +1,7 @@
+awsSecretName=ml-pipeline-aws-secret
+awsAccessKeySecretKey=accesskey
+awsSecretKeySecretKey=secretkey
+awsRegion=ap-southeast-1
+artifactRepositoryBucket=mlpipeline
+artifactRepositoryKeyPrefix=artifacts
+executorImage=gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance

manifests/kustomize/env/aws/accesskey/params.yaml

@@ -0,0 +1,9 @@
+varReference:
+- path: spec/template/spec/containers/env/value
+  kind: Deployment
+- path: spec/template/spec/containers/env/valueFrom/secretKeyRef/name
+  kind: Deployment
+- path: spec/template/spec/containers/env/valueFrom/secretKeyRef/key
+  kind: Deployment
+- path: data
+  kind: ConfigMap

manifests/kustomize/env/aws/accesskey/viewer-tensorboard-template-configmap.yaml

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ml-pipeline-ui-viewer-template
+data:
+  viewer-tensorboard-template.json: |
+    {
+        "spec": {
+            "containers": [
+                {
+                    "env": [
+                        {
+                            "name": "AWS_ACCESS_KEY_ID",
+                            "valueFrom": {
+                                "secretKeyRef": {
+                                    "name": "$(awsSecretName)",
+                                    "key": "$(awsAccessKeySecretKey)"
+                                }
+                            }
+                        },
+                        {
+                            "name": "AWS_SECRET_ACCESS_KEY",
+                            "valueFrom": {
+                                "secretKeyRef": {
+                                    "name": "$(awsSecretName)",
+                                    "key": "$(awsSecretKeySecretKey)"
+                                }
+                            }
+                        },
+                        {
+                            "name": "AWS_REGION",
+                            "value": "ap-southeast-1"
+                        }
+                    ]
+                }
+            ]
+        }
+    }

manifests/kustomize/env/aws/iam/aws-configurations-patch.yaml

@@ -0,0 +1,78 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: workflow-controller-configmap
+  namespace: kubeflow
+data:
+  config: |
+    {
+    executorImage: $(executorImage),
+    artifactRepository:
+    {
+        archiveLogs: true,
+        s3: {
+            bucket: $(artifactRepositoryBucket),
+            keyPrefix: $(artifactRepositoryKeyPrefix),
+            endpoint: s3.amazonaws.com,
+            insecure: false
+        }
+    }
+    }
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: ml-pipeline-ui
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-ui
+          env:
+            - name: ARGO_ARCHIVE_LOGS
+              value: "true"
+            - name: ARGO_ARCHIVE_ARTIFACTORY
+              value: s3
+            - name: ARGO_ARCHIVE_BUCKETNAME
+              value: $(artifactRepositoryBucket)
+            - name: ARGO_ARCHIVE_PREFIX
+              value: $(artifactRepositoryKeyPrefix)
+            - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH
+              value: /etc/config/viewer-tensorboard-template.json
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/config
+      volumes:
+        - name: config-volume
+          configMap:
+            name: ml-pipeline-ui-viewer-template
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    metadata:
+      annotations:
+        iam.amazonaws.com/role: $(awsIAMRole)
+    spec:
+      containers:
+        - name: ml-pipeline-api-server
+          env:
+            - name: OBJECTSTORECONFIG_BUCKETNAME
+              value: $(artifactRepositoryBucket)
+            - name: MINIO_SERVICE_SERVICE_HOST
+              value: s3.amazonaws.com
+            - name: MINIO_SERVICE_SERVICE_PORT
+              value: "443"
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: workflow-controller
+spec:
+  template:
+    metadata:
+      annotations:
+        iam.amazonaws.com/role: $(awsIAMRole)