kubeflow · k8s-ci-robot · Jun 18, 2020 · Jun 11, 2020 · Jun 11, 2020 · Jun 11, 2020
diff --git a/pipeline/cache/base_v3/cluster-role-binding.yaml b/pipeline/cache/base_v3/cluster-role-binding.yaml
@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubeflow-pipelines-cache-deployer-clusterrolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubeflow-pipelines-cache-deployer-clusterrole
+subjects:
+- kind: ServiceAccount
+  name: kubeflow-pipelines-cache-deployer-sa
diff --git a/pipeline/cache/base_v3/kustomization.yaml b/pipeline/cache/base_v3/kustomization.yaml
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../upstream/base/cache-deployer/cluster-scoped/cache-deployer-clusterrole.yaml
+- cluster-role-binding.yaml
+- ../../upstream/base/cache-deployer
+- ../../upstream/base/cache
diff --git a/pipeline/installs/generic/kustomization.yaml b/pipeline/installs/generic/kustomization.yaml
@@ -0,0 +1,59 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../upstream/base/pipeline/cluster-scoped
+- ../../upstream/base/pipeline
+- virtual-service.yaml
+- ../../metadata-writer/base_v3
+- ../../cache/base_v3
+images:
+- name: gcr.io/ml-pipeline/api-server
+  newTag: 1.0.0-rc.1
+- name: gcr.io/ml-pipeline/persistenceagent
+  newTag: 1.0.0-rc.1
+- name: gcr.io/ml-pipeline/scheduledworkflow
+  newTag: 1.0.0-rc.1
+- name: gcr.io/ml-pipeline/frontend
+  newTag: 1.0.0-rc.1
+- name: gcr.io/ml-pipeline/viewer-crd-controller
+  newTag: 1.0.0-rc.1
+- name: gcr.io/ml-pipeline/visualization-server
+  newTag: 1.0.0-rc.1
+- name: gcr.io/ml-pipeline/metadata-writer
+  newTag: 1.0.0-rc.1
+- name: gcr.io/ml-pipeline/cache-server
+  newTag: 1.0.0-rc.1
+- name: gcr.io/ml-pipeline/cache-deployer
+  newTag: 1.0.0-rc.1
+# Used by Kustomize
+configMapGenerator:
+- name: pipeline-install-config
+  env: params.env
+secretGenerator:
+- name: mysql-secret
+  env: params-db-secret.env
+vars:
+# TODO: replace BUCKET_NAME with a configmap value reference
+- name: BUCKET_NAME
+  objref:
+    kind: ConfigMap
+    name: pipeline-install-config
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.bucketName
+- name: KFP_UI_NAMESPACE
+  objref:
+    kind: Service
+    name: ml-pipeline-ui
+    apiVersion: v1
+  fieldref:
+    fieldpath: metadata.namespace
+- name: KFP_UI_SERVICE
+  objref:
+    kind: Service
+    name: ml-pipeline-ui
+    apiVersion: v1
+  fieldref:
+    fieldpath: metadata.name
+configurations:
+- params.yaml
diff --git a/pipeline/installs/generic/params-db-secret.env b/pipeline/installs/generic/params-db-secret.env
@@ -0,0 +1,2 @@
+username=root
+password=
diff --git a/pipeline/installs/generic/params.env b/pipeline/installs/generic/params.env
@@ -0,0 +1 @@
+bucketName=mlpipeline
diff --git a/pipeline/installs/generic/params.yaml b/pipeline/installs/generic/params.yaml
@@ -0,0 +1,3 @@
+varReference:
+- path: spec/http/route/destination/host
+  kind: VirtualService
diff --git a/pipeline/installs/generic/virtual-service.yaml b/pipeline/installs/generic/virtual-service.yaml
@@ -0,0 +1,21 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: ml-pipeline-ui
+spec:
+  gateways:
+  - kubeflow-gateway
+  hosts:
+  - '*'
+  http:
+  - match:
+    - uri:
+        prefix: /pipeline
+    rewrite:
+      uri: /pipeline
+    route:
+    - destination:
+        host: $(KFP_UI_SERVICE).$(KFP_UI_NAMESPACE).svc.$(clusterDomain)
+        port:
+          number: 80
+    timeout: 300s
diff --git a/pipeline/metadata-writer/base_v3/kustomization.yaml b/pipeline/metadata-writer/base_v3/kustomization.yaml
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../upstream/base/metadata/metadata-writer-deployment.yaml
+  - ../../upstream/base/metadata/metadata-writer-role.yaml
+  - ../../upstream/base/metadata/metadata-writer-rolebinding.yaml
+  - ../../upstream/base/metadata/metadata-writer-sa.yaml
diff --git a/pipeline/mysql/installs/gcp-pd/kustomization.yaml b/pipeline/mysql/installs/gcp-pd/kustomization.yaml
@@ -8,6 +8,7 @@ resources:
 - ../../../upstream/env/platform-agnostic/mysql
 - ../../overlays/application/application.yaml
 - persistent-volume.yaml
+- ../generic/configmap.yaml
 patchesStrategicMerge:
 - persistent-volume-claim.yaml
 images:

diff --git a/pipeline/mysql/installs/generic/configmap.yaml b/pipeline/mysql/installs/generic/configmap.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql-configmap
+data:
+  host: mysql
+  port: "3306"
+  mlmd_db: metadb
+  cache_db: cachedb
+  pipeline_db: mlpipeline
diff --git a/pipeline/mysql/installs/generic/kustomization.yaml b/pipeline/mysql/installs/generic/kustomization.yaml
@@ -7,6 +7,7 @@ commonLabels:
 resources:
 - ../../../upstream/env/platform-agnostic/mysql
 - ../../overlays/application/application.yaml
+- ./configmap.yaml
 images:
 - name: gcr.io/ml-pipeline/mysql
   newTag: '5.6'

diff --git a/stacks/generic/kustomization.yaml b/stacks/generic/kustomization.yaml
@@ -18,6 +18,8 @@ resources:
   - ../../argo/base_v3
   - ../../pipeline/minio/installs/generic
   - ../../pipeline/mysql/installs/generic
+  - ../../pipeline/installs/generic
+  - ../../metadata/v3
   # This package will create a profile resource so it needs to be installed after the profiles CR
   - ../../default-install/base
 configMapGenerator:

diff --git a/...piextensions.k8s.io_v1beta1_customresourcedefinition_scheduledworkflows.kubeflow.org.yaml b/...piextensions.k8s.io_v1beta1_customresourcedefinition_scheduledworkflows.kubeflow.org.yaml
@@ -0,0 +1,18 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: scheduledworkflows.kubeflow.org
+spec:
+  group: kubeflow.org
+  names:
+    kind: ScheduledWorkflow
+    listKind: ScheduledWorkflowList
+    plural: scheduledworkflows
+    shortNames:
+    - swf
+    singular: scheduledworkflow
+  scope: Namespaced
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
diff --git a/.../expected/apiextensions.k8s.io_v1beta1_customresourcedefinition_viewers.kubeflow.org.yaml b/.../expected/apiextensions.k8s.io_v1beta1_customresourcedefinition_viewers.kubeflow.org.yaml
@@ -0,0 +1,18 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: viewers.kubeflow.org
+spec:
+  group: kubeflow.org
+  names:
+    kind: Viewer
+    listKind: ViewerList
+    plural: viewers
+    shortNames:
+    - vi
+    singular: viewer
+  scope: Namespaced
+  versions:
+  - name: v1beta1
+    served: true
+    storage: true
diff --git a/...tacks/examples/alice/test_data/expected/apps_v1_deployment_cache-deployer-deployment.yaml b/...tacks/examples/alice/test_data/expected/apps_v1_deployment_cache-deployer-deployment.yaml
@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: cache-deployer
+  name: cache-deployer-deployment
+  namespace: kubeflow
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cache-deployer
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: cache-deployer
+    spec:
+      containers:
+      - env:
+        - name: NAMESPACE_TO_WATCH
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: gcr.io/ml-pipeline/cache-deployer:1.0.0-rc.1
+        imagePullPolicy: Always
+        name: main
+      restartPolicy: Always
+      serviceAccountName: kubeflow-pipelines-cache-deployer-sa
diff --git a/tests/stacks/examples/alice/test_data/expected/apps_v1_deployment_cache-server.yaml b/tests/stacks/examples/alice/test_data/expected/apps_v1_deployment_cache-server.yaml
@@ -0,0 +1,73 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: cache-server
+  name: cache-server
+  namespace: kubeflow
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cache-server
+  template:
+    metadata:
+      labels:
+        app: cache-server
+    spec:
+      containers:
+      - args:
+        - --db_driver=$(DBCONFIG_DRIVER)
+        - --db_host=$(DBCONFIG_HOST_NAME)
+        - --db_port=$(DBCONFIG_PORT)
+        - --db_name=$(DBCONFIG_DB_NAME)
+        - --db_user=$(DBCONFIG_USER)
+        - --db_password=$(DBCONFIG_PASSWORD)
+        - --namespace_to_watch=$(NAMESPACE_TO_WATCH)
+        env:
+        - name: DBCONFIG_DRIVER
+          value: mysql
+        - name: DBCONFIG_DB_NAME
+          valueFrom:
+            configMapKeyRef:
+              key: cache_db
+              name: mysql-configmap
+        - name: DBCONFIG_HOST_NAME
+          valueFrom:
+            configMapKeyRef:
+              key: host
+              name: mysql-configmap
+        - name: DBCONFIG_PORT
+          valueFrom:
+            configMapKeyRef:
+              key: port
+              name: mysql-configmap
+        - name: DBCONFIG_USER
+          valueFrom:
+            secretKeyRef:
+              key: username
+              name: mysql-secret-fd5gktm75t
+        - name: DBCONFIG_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              key: password
+              name: mysql-secret-fd5gktm75t
+        - name: NAMESPACE_TO_WATCH
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: gcr.io/ml-pipeline/cache-server:1.0.0-rc.1
+        imagePullPolicy: Always
+        name: server
+        ports:
+        - containerPort: 8443
+          name: webhook-api
+        volumeMounts:
+        - mountPath: /etc/webhook/certs
+          name: webhook-tls-certs
+          readOnly: true
+      serviceAccountName: kubeflow-pipelines-cache
+      volumes:
+      - name: webhook-tls-certs
+        secret:
+          secretName: webhook-server-tls
diff --git a/tests/stacks/examples/alice/test_data/expected/apps_v1_deployment_metadata-db.yaml b/tests/stacks/examples/alice/test_data/expected/apps_v1_deployment_metadata-db.yaml
@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    component: db
+    kustomize.component: metadata
+  name: metadata-db
+  namespace: kubeflow
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: db
+      kustomize.component: metadata
+  template:
+    metadata:
+      annotations:
+        sidecar.istio.io/inject: "false"
+      labels:
+        component: db
+        kustomize.component: metadata
+      name: db
+    spec:
+      containers:
+      - args:
+        - --datadir
+        - /var/lib/mysql/datadir
+        envFrom:
+        - configMapRef:
+            name: metadata-db-parameters
+        - secretRef:
+            name: metadata-db-secrets
+        image: mysql:8.0.3
+        name: db-container
+        ports:
+        - containerPort: 3306
+          name: dbapi
+        readinessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -c
+            - mysql -D $$MYSQL_DATABASE -p$$MYSQL_ROOT_PASSWORD -e 'SELECT 1'
+          initialDelaySeconds: 5
+          periodSeconds: 2
+          timeoutSeconds: 1
+        volumeMounts:
+        - mountPath: /var/lib/mysql
+          name: metadata-mysql
+      volumes:
+      - name: metadata-mysql
+        persistentVolumeClaim:
+          claimName: metadata-mysql