diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml index a1984505547..e1575f12279 100644 --- a/.cloudbuild.yaml +++ b/.cloudbuild.yaml @@ -28,7 +28,7 @@ steps: id: 'copyPythonSDKToLatest' waitFor: ['preparePythonSDK'] -# Build the pipeline system images and bootstrapper +# Build the pipeline system images - name: 'debian' entrypoint: '/bin/bash' args: ['-c', 'sed -i -e "s/ARG DATE/ENV DATE \"$(date -u)\"/" /workspace/frontend/Dockerfile'] @@ -57,46 +57,6 @@ steps: args: ['build', '-t', 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA', '-f', '/workspace/backend/Dockerfile.persistenceagent', '/workspace'] id: 'buildPersistenceAgent' -- name: 'gcr.io/cloud-builders/docker' - args: ['build', '--build-arg', 'RELEASE_VERSION=$COMMIT_SHA', - '--build-arg', 'API_SERVER_IMAGE=gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA', - '--build-arg', 'SCHEDULED_WORKFLOW_IMAGE=gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA', - '--build-arg', 'PERSISTENCE_AGENT_IMAGE=gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA', - '--build-arg', 'UI_IMAGE=gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA', - '-t', 'gcr.io/$PROJECT_ID/bootstrapper:$COMMIT_SHA', '/workspace/pipeline'] - id: 'buildBootstrapper' -- name: 'debian' - entrypoint: '/bin/bash' - args: ['-c', 'sed -i -e "s/gcr.io\/ml-pipeline\/bootstrapper:\([a-zA-Z0-9_.-]\)\+/gcr.io\/$PROJECT_ID\/bootstrapper:$COMMIT_SHA/g" /workspace/bootstrapper.yaml'] - id: 'prepareBootstrapperYAML' -- name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', '/workspace/bootstrapper.yaml', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/bootstrapper.yaml'] - id: 'copyBootstrapperYAML' - waitFor: ['prepareBootstrapperYAML'] -- name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', '/workspace/bootstrapper.yaml', 'gs://$PROJECT_ID/builds/latest/bootstrapper.yaml'] - id: 'copyBootstrapperYAMLToLatest' - waitFor: ['prepareBootstrapperYAML'] - -# Copy the uninstaller config -- name: 'debian' - entrypoint: '/bin/bash' - args: ['-c', 'sed -i -e "s/gcr.io\/ml-pipeline\/bootstrapper:\([a-zA-Z0-9_.-]\)\+/gcr.io\/$PROJECT_ID\/bootstrapper:$COMMIT_SHA/g" /workspace/uninstaller.yaml'] - id: 'prepareUninstallerYAML' -- name: 'debian' - entrypoint: '/bin/bash' - args: ['-c', 'sed -i -e "s/args: \[/args: [\n \"--api_image\", \"gcr.io\/$PROJECT_ID\/api-server:$COMMIT_SHA\",\n \"--scheduled_workflow_image\", \"gcr.io\/$PROJECT_ID\/scheduledworkflow:$COMMIT_SHA\",\n \"--persistence_agent_image\", \"gcr.io\/$PROJECT_ID\/persistenceagent:$COMMIT_SHA\",\n \"--ui_image\", \"gcr.io\/$PROJECT_ID\/frontend:$COMMIT_SHA\"\n/g" /workspace/uninstaller.yaml'] - id: 'updateUninstallerYAMLImage' - waitFor: ['prepareUninstallerYAML'] -- name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', '/workspace/uninstaller.yaml', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/uninstaller.yaml'] - id: 'copyUninstallerYAML' - waitFor: ['updateUninstallerYAMLImage'] -- name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', '/workspace/uninstaller.yaml', 'gs://$PROJECT_ID/builds/latest/uninstaller.yaml'] - id: 'copyUninstallerYAMLToLatest' - waitFor: ['prepareUninstallerYAML'] - # Build the Dataflow-based pipeline component images - name: 'gcr.io/cloud-builders/docker' @@ -197,7 +157,6 @@ images: - 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA' - 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA' - 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA' -- 'gcr.io/$PROJECT_ID/bootstrapper:$COMMIT_SHA' # Images for the Dataflow-based pipeline components - 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tf-predict:$COMMIT_SHA' diff --git a/.release.cloudbuild.yaml b/.release.cloudbuild.yaml index 1fb614af519..00eb3071df9 100644 --- a/.release.cloudbuild.yaml +++ b/.release.cloudbuild.yaml @@ -62,47 +62,6 @@ steps: id: 'tagPersistenceagentCommitSHA' waitFor: ['pullPersistenceagent'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/bootstrapper:$COMMIT_SHA'] - id: 'pullBootstrapper' -- name: 'gcr.io/cloud-builders/docker' - entrypoint: '/bin/bash' - args: ['-c', 'printf "FROM gcr.io/$PROJECT_ID/bootstrapper:$COMMIT_SHA\nENV API_SERVER_IMAGE gcr.io/ml-pipeline/api-server:$COMMIT_SHA\nENV SCHEDULED_WORKFLOW_IMAGE gcr.io/ml-pipeline/scheduledworkflow:$COMMIT_SHA\nENV PERSISTENCE_AGENT_IMAGE gcr.io/ml-pipeline/persistenceagent:$COMMIT_SHA\nENV UI_IMAGE gcr.io/ml-pipeline/frontend:$COMMIT_SHA" > Dockerfile; docker build -t gcr.io/ml-pipeline/bootstrapper:$COMMIT_SHA .'] - id: 'buildBootstrapper' - waitFor: ['pullBootstrapper'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/ml-pipeline/bootstrapper:$COMMIT_SHA', 'gcr.io/ml-pipeline/bootstrapper:$TAG_NAME'] - id: 'tagBootstrapperVersionNumber' - waitFor: ['buildBootstrapper'] - -# Copy the bootstrapper config -- name: 'debian' - entrypoint: '/bin/bash' - args: ['-c', 'sed -i -e "s/gcr.io\/ml-pipeline\/bootstrapper:\([a-zA-Z0-9_.-]\)\+/gcr.io\/ml-pipeline\/bootstrapper:$TAG_NAME/g" /workspace/bootstrapper.yaml'] - id: 'prepareBootstrapperYAML' -- name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', '/workspace/bootstrapper.yaml', 'gs://ml-pipeline/release/$TAG_NAME/bootstrapper.yaml'] - id: 'copyBootstrapperYAML' - waitFor: ['prepareBootstrapperYAML'] -- name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', '/workspace/bootstrapper.yaml', 'gs://ml-pipeline/release/latest/bootstrapper.yaml'] - id: 'copyBootstrapperYAMLToLatest' - waitFor: ['prepareBootstrapperYAML'] - -# Copy the uninstaller config -- name: 'debian' - entrypoint: '/bin/bash' - args: ['-c', 'sed -i -e "s/gcr.io\/ml-pipeline\/bootstrapper:\([a-zA-Z0-9_.-]\)\+/gcr.io\/ml-pipeline\/bootstrapper:$TAG_NAME/g" /workspace/uninstaller.yaml'] - id: 'prepareUninstallerYAML' -- name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', '/workspace/uninstaller.yaml', 'gs://ml-pipeline/release/$TAG_NAME/uninstaller.yaml'] - id: 'copyUninstallerYAML' - waitFor: ['prepareUninstallerYAML'] -- name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', '/workspace/uninstaller.yaml', 'gs://ml-pipeline/release/latest/uninstaller.yaml'] - id: 'copyUninstallerYAMLToLatest' - waitFor: ['prepareUninstallerYAML'] - # Copy the Python SDK - name: 'gcr.io/cloud-builders/gsutil' args: ['cp', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz', '/workspace/'] @@ -140,7 +99,5 @@ images: - 'gcr.io/ml-pipeline/scheduledworkflow:$COMMIT_SHA' - 'gcr.io/ml-pipeline/persistenceagent:$TAG_NAME' - 'gcr.io/ml-pipeline/persistenceagent:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/bootstrapper:$TAG_NAME' -- 'gcr.io/ml-pipeline/bootstrapper:$COMMIT_SHA' timeout: '1200s' diff --git a/.travis.yml b/.travis.yml index c1737469aaa..f3b4b5161b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,4 +60,4 @@ matrix: # Component SDK tests - cd $TRAVIS_BUILD_DIR/component_sdk/python - - ./run_test.sh \ No newline at end of file + - ./run_test.sh diff --git a/backend/Dockerfile.viewercontroller b/backend/Dockerfile.viewercontroller index 41c71a4bcee..7a3069f7bb8 100644 --- a/backend/Dockerfile.viewercontroller +++ b/backend/Dockerfile.viewercontroller @@ -1,17 +1,22 @@ FROM golang:alpine as builder -RUN apk --no-cache add curl git dep +RUN apk update && apk upgrade +RUN apk add --no-cache git gcc musl-dev -WORKDIR /go/src/github.com/kubeflow/pipelines +WORKDIR /src/github.com/kubeflow/pipelines COPY . . +RUN go mod vendor RUN go build -o /bin/controller backend/src/crd/controller/viewer/*.go FROM alpine +WORKDIR /src +COPY --from=builder /src/github.com/kubeflow/pipelines/vendor vendor + WORKDIR /bin COPY --from=builder /bin/controller /bin/controller -COPY --from=builder /go/src/github.com/kubeflow/pipelines/third_party/license.txt /bin/license.txt +COPY --from=builder /src/github.com/kubeflow/pipelines/third_party/license.txt /bin/license.txt RUN chmod +x /bin/controller CMD /bin/controller -alsologtostderr=true \ No newline at end of file diff --git a/bootstrapper.yaml b/bootstrapper.yaml deleted file mode 100644 index 10c7b70459c..00000000000 --- a/bootstrapper.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This manifest file creates a cluster admin role and bind to default service account in default namespace, -# before starting the deployment job. This is needed since the deployment job runs a pod that creates -# more cluster roles in the cluster. The pod is ran by the default service account so this account needs -# additional permission to finish the deployment job successfully. -# -# If you think the default service account shouldn't need that level of RBAC permission in the future, -# it's safe to delete the mlpipeline-admin-role-binding after deployment succeeded: -# kubectl delete ClusterRoleBinding mlpipeline-admin-role-binding -# Or you can run the deployment job with service account you would like to use. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: mlpipeline-deploy-admin -rules: -- apiGroups: - - '*' - resources: - - '*' - verbs: - - '*' -- nonResourceURLs: - - '*' - verbs: - - '*' - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - creationTimestamp: null - name: mlpipeline-admin-role-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: mlpipeline-deploy-admin -subjects: -- kind: ServiceAccount - name: default - namespace: default - ---- -apiVersion: batch/v1 -kind: Job -metadata: - generateName: deploy-ml-pipeline- -spec: - backoffLimit: 1 - template: - metadata: - name: deploy-ml-pipeline - spec: - containers: - - name: deploy - image: gcr.io/ml-pipeline/bootstrapper:0.1.4 - imagePullPolicy: 'Always' - # Additional parameter available: - args: [ - # "--namespace", "foo", - # "--report_usage", "false", - # "--uninstall", - ] - restartPolicy: Never diff --git a/pipeline/Dockerfile b/pipeline/Dockerfile deleted file mode 100644 index b28af834b7c..00000000000 --- a/pipeline/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -# This is a dockerfile for an image with kubectl and ksonnet installed -# Available at gcr.io/ml-pipeline/bootstrapper - -FROM google/cloud-sdk - -RUN apt-get update -y -RUN apt-get install -y curl - -# Install ksonnet -ARG KS_VERSION="0.13.0" -RUN curl -LO https://github.com/ksonnet/ksonnet/releases/download/v${KS_VERSION}/ks_${KS_VERSION}_linux_amd64.tar.gz -RUN tar -xzf ks_${KS_VERSION}_linux_amd64.tar.gz -RUN chmod +x ./ks_${KS_VERSION}_linux_amd64/ks -RUN mv ./ks_${KS_VERSION}_linux_amd64/ks /usr/local/bin/ - -# Set USER env variable to workaround a known issue -# https://github.com/ksonnet/ksonnet/blob/master/docs/troubleshooting.md#error-user-current-not-implemented-on-linuxamd64 -ENV USER=default - -# Copy bootstrapper files -COPY . /pipeline/ -RUN chmod +x /pipeline/bootstrapper.sh -RUN chmod +x /pipeline/deploy.sh - -# install uuidgen. Used for reporting usage -RUN apt-get install -y uuid-runtime - -# Specify the release version and images as a build time arguments so that it can be specified in the `docker build ...` command. -ARG RELEASE_VERSION -ARG API_SERVER_IMAGE -ARG SCHEDULED_WORKFLOW_IMAGE -ARG PERSISTENCE_AGENT_IMAGE -ARG UI_IMAGE - -# Additionally specify the release version and images as environment variables so that they are available to the -# `deploy.sh` script at runtime. The default values of each environment variable is the value of the build arg. -ENV RELEASE_VERSION ${RELEASE_VERSION} -ENV API_SERVER_IMAGE ${API_SERVER_IMAGE} -ENV SCHEDULED_WORKFLOW_IMAGE ${SCHEDULED_WORKFLOW_IMAGE} -ENV PERSISTENCE_AGENT_IMAGE ${PERSISTENCE_AGENT_IMAGE} -ENV UI_IMAGE ${UI_IMAGE} - -ENTRYPOINT ["/pipeline/deploy.sh"] -CMD [] diff --git a/pipeline/OWNERS b/pipeline/OWNERS deleted file mode 100644 index 197efbf5c2e..00000000000 --- a/pipeline/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -approvers: - - vicaire - - IronPan -reviewers: - - vicaire - - IronPan diff --git a/pipeline/bootstrapper.sh b/pipeline/bootstrapper.sh deleted file mode 100755 index ff4e2af5360..00000000000 --- a/pipeline/bootstrapper.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -e - -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# When kubectl is installed in a k8s pod, it uses env variables instead of config file to talk with master node -# https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#accessing-the-service. -# The kubeconfig file is empty. -# Generate kubeconfig file based on the env variable for ksonnet to connect to K8s API inside a pod. -mkdir /root/.kube - -export KUBERNETES_TOKEN=`cat /var/run/secrets/kubernetes.io/serviceaccount/token` - -cat >/root/.kube/config < ${APP_DIR}/tmp.yaml -mv ${APP_DIR}/tmp.yaml ${APP_DIR}/app.yaml - - -kubectl get ns ${NAMESPACE} &>/dev/null -if [ $? == 0 ]; then - echo "namespace ${NAMESPACE} exist" -else - echo "Creating a new kubernetes namespace ..." - kubectl create ns ${NAMESPACE} -fi - -# Generate a ksonnet component manifest and assign parameters -( cd ${APP_DIR} && ks generate pipeline pipeline ) -( cd ${APP_DIR} && ks env set default --namespace ${NAMESPACE} ) -( cd ${APP_DIR} && ks param set pipeline apiImage ${API_SERVER_IMAGE} ) -( cd ${APP_DIR} && ks param set pipeline scheduledWorkflowImage ${SCHEDULED_WORKFLOW_IMAGE} ) -( cd ${APP_DIR} && ks param set pipeline persistenceAgentImage ${PERSISTENCE_AGENT_IMAGE} ) -( cd ${APP_DIR} && ks param set pipeline uiImage ${UI_IMAGE} ) -( cd ${APP_DIR} && ks param set pipeline deployArgo ${DEPLOY_ARGO} ) -( cd ${APP_DIR} && ks param set pipeline reportUsage ${REPORT_USAGE} ) -( cd ${APP_DIR} && ks param set pipeline usageId $(uuidgen) ) - -# Get current active service account and create a user-gcp-sa secret with the service key -if [ "$PLATFORM" = "gcp" ]; then - gcloud services enable servicemanagement.googleapis.com iam.googleapis.com - SERVICE_ACCOUNT=$(gcloud auth list --filter=status:ACTIVE --format="value(account)") - gcloud iam service-accounts keys create service_account_key --iam-account $SERVICE_ACCOUNT - kubectl create secret generic --namespace=${NAMESPACE} user-gcp-sa --from-file=user-gcp-sa.json=service_account_key - rm service_account_key -fi - -if [ "$WITH_KUBEFLOW" = true ]; then - # v0.2 non-gke deploy script doesn't create a namespace. This would be fixed in the later version. - # https://github.com/kubeflow/kubeflow/blob/master/scripts/deploy.sh#L43 - mkdir -p ${KF_DIR} - # We use kubeflow v0.3.2 by default - KUBEFLOW_VERSION=${KUBEFLOW_VERSION:-"v0.3.2"} - (cd ${KF_DIR} && curl -L -o kubeflow.tar.gz https://github.com/kubeflow/kubeflow/archive/${KUBEFLOW_VERSION}.tar.gz) - tar -xzf ${KF_DIR}/kubeflow.tar.gz -C ${KF_DIR} - KUBEFLOW_REPO=$(find ${KF_DIR} -maxdepth 1 -type d -name "kubeflow*") - if [[ ${REPORT_USAGE} != "true" ]]; then - (cd ${KUBEFLOW_REPO} && sed -i -e 's/--reportUsage\=true/--reportUsage\=false/g' scripts/util.sh) - fi - (${KUBEFLOW_REPO}/scripts/kfctl.sh init ${KFAPP} --platform none) - (cd ${KFAPP} && ${KUBEFLOW_REPO}/scripts/kfctl.sh generate k8s && ${KUBEFLOW_REPO}/scripts/kfctl.sh apply k8s) -fi - -if ${UNINSTALL} ; then - ( cd ${APP_DIR} && ks delete default) - if [ "$PLATFORM" = "gcp" ]; then - kubectl delete secret --namespace=${NAMESPACE} user-gcp-sa - fi - if [ "$WITH_KUBEFLOW" = true ]; then - KUBEFLOW_REPO=$(find ${KF_DIR} -maxdepth 1 -type d -name "kubeflow*") - # Uninstall Kubeflow - (cd ${KUBEFLOW_REPO}${KFAPP} && ks delete default) - fi - exit 0 -fi - -# Install ML pipeline -( cd ${APP_DIR} && ks apply default -c pipeline) - -# Wait for service to be ready - -MAX_ATTEMPT=60 -READY_KEYWORD="\"apiServerReady\":true" -CA_CERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt -TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) - -# probing the UI healthz/ status until it's ready. Timeout after 4 minutes -echo "Waiting for ML pipeline to be ready..." -for i in $(seq 1 ${MAX_ATTEMPT}) -do - echo -n . - UI_STATUS=`curl -sS --cacert $CA_CERT -H "Authorization: Bearer $TOKEN" https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}/api/v1/namespaces/${NAMESPACE}/services/ml-pipeline-ui:80/proxy/apis/v1beta1/healthz` - echo $UI_STATUS | grep -q ${READY_KEYWORD} && s=0 && break || s=$? && sleep 4 -done - -if [[ $s != 0 ]] - then echo "ML Pipeline not start successfully after 4 minutes. Timeout..." && exit $s -else - echo "ML Pipeline Is Now Ready" && exit $s -fi diff --git a/pipeline/license.txt b/pipeline/license.txt deleted file mode 100644 index d1eb5cae75d..00000000000 --- a/pipeline/license.txt +++ /dev/null @@ -1,614 +0,0 @@ --------------------------------------------------------------------------------- - kubectl LICENSE --------------------------------------------------------------------------------- - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --------------------------------------------------------------------------------- - ksonnet LICENSE --------------------------------------------------------------------------------- - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --------------------------------------------------------------------------------- - kubeflow LICENSE --------------------------------------------------------------------------------- - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/pipeline/pipeline/all.libsonnet b/pipeline/pipeline/all.libsonnet deleted file mode 100644 index 2a8f6c8741b..00000000000 --- a/pipeline/pipeline/all.libsonnet +++ /dev/null @@ -1,40 +0,0 @@ -{ - parts(_env, _params):: { - local params = _env + _params, - - local argo = import "pipeline/pipeline/argo.libsonnet", - local minio = import "pipeline/pipeline/minio.libsonnet", - local mysql = import "pipeline/pipeline/mysql.libsonnet", - local pipeline_apiserver = import "pipeline/pipeline/pipeline-apiserver.libsonnet", - local pipeline_scheduledworkflow = import "pipeline/pipeline/pipeline-scheduledworkflow.libsonnet", - local pipeline_persistenceagent = import "pipeline/pipeline/pipeline-persistenceagent.libsonnet", - local pipeline_ui = import "pipeline/pipeline/pipeline-ui.libsonnet", - local spartakus = import "pipeline/pipeline/spartakus.libsonnet", - - local name = params.name, - local namespace = params.namespace, - local apiImage = params.apiImage, - local scheduledWorkflowImage = params.scheduledWorkflowImage, - local persistenceAgentImage = params.persistenceAgentImage, - local uiImage = params.uiImage, - local mysqlImage = params.mysqlImage, - local minioImage = params.minioImage, - local deployArgo = params.deployArgo, - local reportUsage = params.reportUsage, - local usageId = params.usageId, - reporting:: if (reportUsage == true) || (reportUsage == "true") then - spartakus.all(namespace, usageId) - else [], - argo:: if (deployArgo == true) || (deployArgo == "true") then - argo.parts(namespace).all - else [], - all:: minio.all(namespace, minioImage) + - mysql.all(namespace, mysqlImage) + - pipeline_apiserver.all(namespace, apiImage) + - pipeline_scheduledworkflow.all(namespace, scheduledWorkflowImage) + - pipeline_persistenceagent.all(namespace, persistenceAgentImage) + - pipeline_ui.all(namespace, uiImage) + - $.parts(_env, _params).argo + - $.parts(_env, _params).reporting, - }, -} diff --git a/pipeline/pipeline/argo.libsonnet b/pipeline/pipeline/argo.libsonnet deleted file mode 100644 index 04ca639b456..00000000000 --- a/pipeline/pipeline/argo.libsonnet +++ /dev/null @@ -1,429 +0,0 @@ -{ - parts(namespace):: { - all:: [ - $.parts(namespace).crd, - $.parts(namespace).config, - $.parts(namespace).deploy, - $.parts(namespace).deployUi, - $.parts(namespace).uiService, - $.parts(namespace).serviceAccount, - $.parts(namespace).role, - $.parts(namespace).roleBinding, - $.parts(namespace).uiServiceAccount, - $.parts(namespace).uiRole, - $.parts(namespace).uiRoleBinding, - ], - crd: { - apiVersion: "apiextensions.k8s.io/v1beta1", - kind: "CustomResourceDefinition", - metadata: { - name: "workflows.argoproj.io", - }, - spec: { - group: "argoproj.io", - names: { - kind: "Workflow", - listKind: "WorkflowList", - plural: "workflows", - shortNames: [ - "wf", - ], - singular: "workflow", - }, - scope: "Namespaced", - version: "v1beta1", - }, - }, // crd - - // Deploy the controller - deploy: { - apiVersion: "extensions/v1beta1", - kind: "Deployment", - labels: { - app: "workflow-controller", - }, - metadata: { - name: "workflow-controller", - namespace: namespace, - }, - spec: { - progressDeadlineSeconds: 600, - replicas: 1, - revisionHistoryLimit: 10, - selector: { - matchLabels: { - app: "workflow-controller", - }, - }, - strategy: { - rollingUpdate: { - maxSurge: "25%", - maxUnavailable: "25%", - }, - type: "RollingUpdate", - }, - template: { - metadata: { - creationTimestamp: null, - labels: { - app: "workflow-controller", - }, - }, - spec: { - containers: [ - { - args: [ - "--configmap", - "workflow-controller-configmap", - ], - command: [ - "workflow-controller", - ], - env: [ - { - name: "ARGO_NAMESPACE", - valueFrom: { - fieldRef: { - apiVersion: "v1", - fieldPath: "metadata.namespace", - }, - }, - }, - ], - image: "argoproj/workflow-controller:v2.2.0", - imagePullPolicy: "IfNotPresent", - name: "workflow-controller", - resources: {}, - terminationMessagePath: "/dev/termination-log", - terminationMessagePolicy: "File", - }, - ], - dnsPolicy: "ClusterFirst", - restartPolicy: "Always", - schedulerName: "default-scheduler", - securityContext: {}, - serviceAccount: "argo", - serviceAccountName: "argo", - terminationGracePeriodSeconds: 30, - }, - }, - }, - }, // deploy - - - deployUi: { - apiVersion: "extensions/v1beta1", - kind: "Deployment", - metadata: { - labels: { - app: "argo-ui", - }, - name: "argo-ui", - namespace: namespace, - }, - spec: { - progressDeadlineSeconds: 600, - replicas: 1, - revisionHistoryLimit: 10, - selector: { - matchLabels: { - app: "argo-ui", - }, - }, - strategy: { - rollingUpdate: { - maxSurge: "25%", - maxUnavailable: "25%", - }, - type: "RollingUpdate", - }, - template: { - metadata: { - creationTimestamp: null, - labels: { - app: "argo-ui", - }, - }, - spec: { - containers: [ - { - env: [ - { - name: "ARGO_NAMESPACE", - valueFrom: { - fieldRef: { - apiVersion: "v1", - fieldPath: "metadata.namespace", - }, - }, - }, - { - name: "IN_CLUSTER", - value: "true", - }, - ], - image: "argoproj/argoui:v2.2.0", - imagePullPolicy: "IfNotPresent", - name: "argo-ui", - resources: {}, - terminationMessagePath: "/dev/termination-log", - terminationMessagePolicy: "File", - }, - ], - dnsPolicy: "ClusterFirst", - restartPolicy: "Always", - schedulerName: "default-scheduler", - securityContext: {}, - serviceAccount: "argo-ui", - serviceAccountName: "argo-ui", - terminationGracePeriodSeconds: 30, - readinessProbe: { - httpGet: { - path: "/", - port: 8001, - }, - }, - }, - }, - }, - }, // deployUi - - uiService: { - apiVersion: "v1", - kind: "Service", - metadata: { - labels: { - app: "argo-ui", - }, - name: "argo-ui", - namespace: namespace, - }, - spec: { - ports: [ - { - port: 80, - targetPort: 8001, - }, - ], - selector: { - app: "argo-ui", - }, - sessionAffinity: "None", - type: "NodePort", - }, - }, - - config: { - apiVersion: "v1", - // The commented out section creates a default artifact repository - // This section is not deleted because we might need it in the future. - // And it takes time to get this string right. - //data: { - // config: "executorImage: argoproj/argoexec:v2.2.0\nartifactRepository:\n s3: - // \n bucket: mlpipeline\n endpoint: minio-service.kubeflow:9000\n insecure: true - // \n accessKeySecret:\n name: mlpipeline-minio-artifact\n key: accesskey\n secretKeySecret: - // \n name: mlpipeline-minio-artifact\n key: secretkey" - //}, - data: { - config: "executorImage: argoproj/argoexec:v2.2.0", - }, - kind: "ConfigMap", - metadata: { - name: "workflow-controller-configmap", - namespace: namespace, - }, - }, - - serviceAccount: { - apiVersion: "v1", - kind: "ServiceAccount", - metadata: { - name: "argo", - namespace: namespace, - }, - }, // service account - - // Keep in sync with https://github.com/argoproj/argo/blob/master/cmd/argo/commands/const.go#L20 - // Permissions need to be cluster wide for the workflow controller to be able to process workflows - // in other namespaces. We could potentially use the ConfigMap of the workflow-controller to - // scope it to a particular namespace in which case we might be able to restrict the permissions - // to a particular namespace. - role: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "ClusterRole", - metadata: { - labels: { - app: "argo", - }, - name: "argo", - namespace: namespace, - }, - rules: [ - { - apiGroups: [""], - resources: [ - "pods", - "pods/exec", - ], - verbs: [ - "create", - "get", - "list", - "watch", - "update", - "patch", - ], - }, - { - apiGroups: [""], - resources: [ - "secrets", - ], - verbs: [ - "get", - ], - }, - { - apiGroups: [""], - resources: [ - "configmaps", - ], - verbs: [ - "get", - "watch", - "list", - ], - }, - { - apiGroups: [ - "", - ], - resources: [ - "persistentvolumeclaims", - ], - verbs: [ - "create", - "delete", - ], - }, - { - apiGroups: [ - "argoproj.io", - ], - resources: [ - "workflows", - ], - verbs: [ - "get", - "list", - "watch", - "update", - "patch", - ], - }, - ], - }, // operator-role - - roleBinding:: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "ClusterRoleBinding", - metadata: { - labels: { - app: "argo", - }, - name: "argo", - namespace: namespace, - }, - roleRef: { - apiGroup: "rbac.authorization.k8s.io", - kind: "ClusterRole", - name: "argo", - }, - subjects: [ - { - kind: "ServiceAccount", - name: "argo", - namespace: namespace, - }, - ], - }, // role binding - - uiServiceAccount: { - apiVersion: "v1", - kind: "ServiceAccount", - metadata: { - name: "argo-ui", - namespace: namespace, - }, - }, // service account - - // Keep in sync with https://github.com/argoproj/argo/blob/master/cmd/argo/commands/const.go#L44 - // Permissions need to be cluster wide for the workflow controller to be able to process workflows - // in other namespaces. We could potentially use the ConfigMap of the workflow-controller to - // scope it to a particular namespace in which case we might be able to restrict the permissions - // to a particular namespace. - uiRole: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "ClusterRole", - metadata: { - labels: { - app: "argo", - }, - name: "argo-ui", - namespace: namespace, - }, - rules: [ - { - apiGroups: [""], - resources: [ - "pods", - "pods/exec", - "pods/log", - ], - verbs: [ - "get", - "list", - "watch", - ], - }, - { - apiGroups: [ - "argoproj.io", - ], - resources: [ - "workflows", - ], - verbs: [ - "get", - "list", - "watch", - ], - }, - ], - }, // operator-role - - uiRoleBinding:: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "ClusterRoleBinding", - metadata: { - labels: { - app: "argo-ui", - }, - name: "argo-ui", - namespace: namespace, - }, - roleRef: { - apiGroup: "rbac.authorization.k8s.io", - kind: "ClusterRole", - name: "argo-ui", - }, - subjects: [ - { - kind: "ServiceAccount", - name: "argo-ui", - namespace: namespace, - }, - ], - }, // role binding - }, // parts -} diff --git a/pipeline/pipeline/minio.libsonnet b/pipeline/pipeline/minio.libsonnet deleted file mode 100644 index 3c1ebbcaa48..00000000000 --- a/pipeline/pipeline/minio.libsonnet +++ /dev/null @@ -1,131 +0,0 @@ -{ - all(namespace, minioImage):: [ - $.parts(namespace).pvc, - $.parts(namespace).service, - $.parts(namespace).deploy(minioImage), - $.parts(namespace).secret, - ], - - parts(namespace):: { - pvc: { - apiVersion: "v1", - kind: "PersistentVolumeClaim", - metadata: { - name: "minio-pv-claim", - namespace: namespace, - }, - spec: { - accessModes: [ - "ReadWriteOnce", - ], - resources: { - requests: { - storage: "10Gi", - }, - }, - }, - }, //pvc - - service: { - apiVersion: "v1", - kind: "Service", - metadata: { - name: "minio-service", - namespace: namespace, - }, - spec: { - ports: [ - { - port: 9000, - targetPort: 9000, - protocol: "TCP", - }, - ], - selector: { - app: "minio", - }, - }, - status: { - loadBalancer: {}, - }, - }, //service - - deploy(image): { - apiVersion: "apps/v1beta1", - kind: "Deployment", - metadata: { - name: "minio", - namespace: namespace, - }, - spec: { - strategy: { - type: "Recreate", - }, - template: { - metadata: { - labels: { - app: "minio", - }, - }, - spec: { - volumes: [ - { - name: "data", - persistentVolumeClaim: { - claimName: "minio-pv-claim", - }, - }, - ], - containers: [ - { - name: "minio", - volumeMounts: [ - { - name: "data", - mountPath: "/data", - }, - ], - image: image, - args: [ - "server", - "/data", - ], - env: [ - { - name: "MINIO_ACCESS_KEY", - value: "minio", - }, - { - name: "MINIO_SECRET_KEY", - value: "minio123", - }, - ], - ports: [ - { - containerPort: 9000, - }, - ], - }, - ], - }, - }, - }, - }, // deploy - - // The motivation behind the minio secret creation is that argo workflows depend on this secret to - // store the artifact in minio. - secret: { - apiVersion: "v1", - kind: "Secret", - metadata: { - name: "mlpipeline-minio-artifact", - namespace: namespace, - }, - type: "Opaque", - data: { - accesskey: std.base64("minio"), - secretkey: std.base64("minio123"), - }, - }, // secret - }, // parts -} diff --git a/pipeline/pipeline/mysql.libsonnet b/pipeline/pipeline/mysql.libsonnet deleted file mode 100644 index 1cd55aa4ad8..00000000000 --- a/pipeline/pipeline/mysql.libsonnet +++ /dev/null @@ -1,111 +0,0 @@ -{ - all(namespace, mysqlImage):: [ - $.parts(namespace).pvc, - $.parts(namespace).service, - $.parts(namespace).deploy(mysqlImage), - ], - parts(namespace):: { - pvc: { - apiVersion: "v1", - kind: "PersistentVolumeClaim", - metadata: { - name: "mysql-pv-claim", - namespace: namespace, - }, - spec: { - accessModes: [ - "ReadWriteOnce", - ], - resources: { - requests: { - storage: "10Gi", - }, - }, - }, - }, //pvc - - service: { - apiVersion: "v1", - kind: "Service", - metadata: { - name: "mysql", - namespace: namespace, - }, - spec: { - ports: [ - { - port: 3306, - }, - ], - selector: { - app: "mysql", - }, - }, - status: { - loadBalancer: {}, - }, - }, //service - - deploy(image): { - apiVersion: "apps/v1beta2", - kind: "Deployment", - metadata: { - name: "mysql", - namespace: namespace, - }, - spec: { - selector: { - matchLabels: { - app: "mysql", - }, - }, - strategy: { - type: "Recreate", - }, - template: { - metadata: { - labels: { - app: "mysql", - }, - }, - spec: { - containers: [ - { - // Before bumping the mysql version, make sure version is also available for - // Google Cloud SQL. As of Dec 2018, only mysql 5.6 or 5.7 is supported in Cloud SQL. - image: image, - name: "mysql", - env: [ - { - name: "MYSQL_ALLOW_EMPTY_PASSWORD", - value: "true", - }, - ], - ports: [ - { - containerPort: 3306, - name: "mysql", - }, - ], - volumeMounts: [ - { - name: "mysql-persistent-storage", - mountPath: "/var/lib/mysql", - }, - ], - }, - ], - volumes: [ - { - name: "mysql-persistent-storage", - persistentVolumeClaim: { - claimName: "mysql-pv-claim", - }, - }, - ], - }, - }, - }, - }, //deploy - }, //parts -} diff --git a/pipeline/pipeline/parts.yaml b/pipeline/pipeline/parts.yaml deleted file mode 100644 index 780e16fcf01..00000000000 --- a/pipeline/pipeline/parts.yaml +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2018 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -{ - name: "pipeline", - apiVersion: "0.0.1", - kind: "ksonnet.io/parts", - description: "Prototypes for running ML pipeline.\n", - author: "kubeflow-team ", - contributors: [ - { - name: "Kubeflow Team", - email: "kubeflow-discuss@googlegroups.com", - }, - ], - repository: { - type: "git", - url: "https://github.com/kubeflow/pipelines", - }, - bugs: { - url: "https://github.com/kubeflow/pipelines/issues", - }, - keywords: [ - "pipeline", - ], - quickStart: { - prototype: "io.ksonnet.pkg.pipeline", - componentName: "pipeline", - flags: { - name: "pipeline", - namespace: "default", - }, - comment: "Deploy ML pipeline", - }, - license: "Apache 2.0", -} diff --git a/pipeline/pipeline/pipeline-apiserver.libsonnet b/pipeline/pipeline/pipeline-apiserver.libsonnet deleted file mode 100644 index bb0224a2630..00000000000 --- a/pipeline/pipeline/pipeline-apiserver.libsonnet +++ /dev/null @@ -1,328 +0,0 @@ -{ - all(namespace, apiImage):: [ - $.parts(namespace).serviceAccount, - $.parts(namespace).roleBinding, - $.parts(namespace).role, - $.parts(namespace).service, - $.parts(namespace).deploy(apiImage), - $.parts(namespace).pipelineRunnerServiceAccount, - $.parts(namespace).pipelineRunnerRole, - $.parts(namespace).pipelineRunnerRoleBinding, - ], - - parts(namespace):: { - serviceAccount: { - apiVersion: "v1", - kind: "ServiceAccount", - metadata: { - name: "ml-pipeline", - namespace: namespace, - }, - }, // service account - - roleBinding:: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "RoleBinding", - metadata: { - labels: { - app: "ml-pipeline", - }, - name: "ml-pipeline", - namespace: namespace, - }, - roleRef: { - apiGroup: "rbac.authorization.k8s.io", - kind: "Role", - name: "ml-pipeline", - }, - subjects: [ - { - kind: "ServiceAccount", - name: "ml-pipeline", - namespace: namespace, - }, - ], - }, // role binding - - role: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "Role", - metadata: { - labels: { - app: "ml-pipeline", - }, - name: "ml-pipeline", - namespace: namespace, - }, - rules: [ - { - apiGroups: [ - "argoproj.io", - ], - resources: [ - "workflows", - ], - verbs: [ - "create", - "get", - "list", - "watch", - "update", - "patch", - "delete", - ], - }, - { - apiGroups: [ - "kubeflow.org", - ], - resources: [ - "scheduledworkflows", - ], - verbs: [ - "create", - "get", - "list", - "update", - "patch", - "delete", - ], - }, - ], - }, // role - - service: { - apiVersion: "v1", - kind: "Service", - metadata: { - labels: { - app: "ml-pipeline", - }, - name: "ml-pipeline", - namespace: namespace, - }, - spec: { - ports: [ - { - port: 8888, - targetPort: 8888, - protocol: "TCP", - name: "http", - }, - { - port: 8887, - targetPort: 8887, - protocol: "TCP", - name: "grpc", - }, - ], - selector: { - app: "ml-pipeline", - }, - }, - status: { - loadBalancer: {}, - }, - }, //service - - deploy(image): { - apiVersion: "apps/v1beta2", - kind: "Deployment", - metadata: { - labels: { - app: "ml-pipeline", - }, - name: "ml-pipeline", - namespace: namespace, - }, - spec: { - selector: { - matchLabels: { - app: "ml-pipeline", - }, - }, - template: { - metadata: { - labels: { - app: "ml-pipeline", - }, - }, - spec: { - containers: [ - { - name: "ml-pipeline-api-server", - image: image, - imagePullPolicy: "Always", - ports: [ - { - containerPort: 8888, - }, - { - containerPort: 8887, - }, - ], - env: [ - { - name: "POD_NAMESPACE", - valueFrom: { - fieldRef: { - fieldPath: "metadata.namespace", - }, - }, - }, - ], - }, - ], - serviceAccountName: "ml-pipeline", - }, - }, - }, - }, // deploy - - pipelineRunnerServiceAccount: { - apiVersion: "v1", - kind: "ServiceAccount", - metadata: { - name: "pipeline-runner", - namespace: namespace, - }, - }, // service account - - - // Grant admin permission so the pipeline can launch any resource in the cluster. - pipelineRunnerRole: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "ClusterRole", - metadata: { - labels: { - app: "pipeline-runner", - }, - name: "pipeline-runner", - namespace: namespace, - }, - rules: [ - { - apiGroups: [""], - resources: [ - "secrets", - ], - verbs: [ - "get", - ], - }, - { - apiGroups: [""], - resources: [ - "configmaps", - ], - verbs: [ - "get", - "watch", - "list", - ], - }, - { - apiGroups: [ - "", - ], - resources: [ - "persistentvolumeclaims", - ], - verbs: [ - "create", - "delete", - ], - }, - { - apiGroups: [ - "argoproj.io", - ], - resources: [ - "workflows", - ], - verbs: [ - "get", - "list", - "watch", - "update", - "patch", - ], - }, - { - apiGroups: [ - "", - ], - resources: [ - "pods", - "pods/exec", - "pods/log", - "services", - ], - verbs: [ - "*", - ], - }, - { - apiGroups: [ - "", - "apps", - "extensions", - ], - resources: [ - "deployments", - "replicasets", - ], - verbs: [ - "*", - ], - }, - { - apiGroups: [ - "kubeflow.org", - ], - resources: [ - "*", - ], - verbs: [ - "*", - ], - }, - { - apiGroups: [ - "batch", - ], - resources: [ - "jobs", - ], - verbs: [ - "*", - ], - }, - ], - }, // operator-role - - pipelineRunnerRoleBinding:: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "ClusterRoleBinding", - metadata: { - labels: { - app: "pipeline-runner", - }, - name: "pipeline-runner", - namespace: namespace, - }, - roleRef: { - apiGroup: "rbac.authorization.k8s.io", - kind: "ClusterRole", - name: "pipeline-runner", - }, - subjects: [ - { - kind: "ServiceAccount", - name: "pipeline-runner", - namespace: namespace, - }, - ], - }, // role binding - }, // parts -} diff --git a/pipeline/pipeline/pipeline-persistenceagent.libsonnet b/pipeline/pipeline/pipeline-persistenceagent.libsonnet deleted file mode 100644 index 8fb2f6d3c4e..00000000000 --- a/pipeline/pipeline/pipeline-persistenceagent.libsonnet +++ /dev/null @@ -1,129 +0,0 @@ -{ - all(namespace, persistenceAgentImage):: [ - $.parts(namespace).serviceAccount, - $.parts(namespace).roleBinding, - $.parts(namespace).role, - $.parts(namespace).deploy(persistenceAgentImage), - ], - - parts(namespace):: { - serviceAccount: { - apiVersion: "v1", - kind: "ServiceAccount", - metadata: { - name: "ml-pipeline-persistenceagent", - namespace: namespace, - }, - }, // service account - - roleBinding:: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "ClusterRoleBinding", - metadata: { - labels: { - app: "ml-pipeline-persistenceagent", - }, - name: "ml-pipeline-persistenceagent", - }, - roleRef: { - apiGroup: "rbac.authorization.k8s.io", - kind: "ClusterRole", - // TODO: These permissions are too broad. This must be fixed. - name: "cluster-admin", - }, - subjects: [ - { - kind: "ServiceAccount", - name: "ml-pipeline-persistenceagent", - namespace: namespace, - }, - ], - }, // role binding - - role: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "ClusterRole", - metadata: { - labels: { - app: "ml-pipeline-persistenceagent", - }, - name: "ml-pipeline-persistenceagent", - namespace: namespace, - }, - rules: [ - { - apiGroups: [ - "argoproj.io", - ], - resources: [ - "workflows", - ], - verbs: [ - "get", - "list", - "watch", - ], - }, - { - apiGroups: [ - "kubeflow.org", - ], - resources: [ - "scheduledworkflows", - ], - verbs: [ - "get", - "list", - "watch", - ], - }, - ], - }, // role - - deploy(image): { - apiVersion: "apps/v1beta2", - kind: "Deployment", - metadata: { - labels: { - app: "ml-pipeline-persistenceagent", - }, - name: "ml-pipeline-persistenceagent", - namespace: namespace, - }, - spec: { - selector: { - matchLabels: { - app: "ml-pipeline-persistenceagent", - }, - }, - template: { - metadata: { - labels: { - app: "ml-pipeline-persistenceagent", - }, - }, - spec: { - containers: [ - { - name: "ml-pipeline-persistenceagent", - image: image, - imagePullPolicy: "Always", - env: [ - { - name: "POD_NAMESPACE", - valueFrom: { - fieldRef: { - fieldPath: "metadata.namespace", - }, - }, - }, - ], - }, - ], - serviceAccountName: "ml-pipeline-persistenceagent", - }, - }, - }, - }, // deploy - }, // parts -} diff --git a/pipeline/pipeline/pipeline-scheduledworkflow.libsonnet b/pipeline/pipeline/pipeline-scheduledworkflow.libsonnet deleted file mode 100644 index 2f34c1c06b5..00000000000 --- a/pipeline/pipeline/pipeline-scheduledworkflow.libsonnet +++ /dev/null @@ -1,159 +0,0 @@ -{ - all(namespace, scheduledWorkflowImage):: [ - $.parts(namespace).serviceAccount, - $.parts(namespace).roleBinding, - $.parts(namespace).role, - $.parts(namespace).deploy(scheduledWorkflowImage), - $.parts(namespace).crd, - ], - - parts(namespace):: { - serviceAccount: { - apiVersion: "v1", - kind: "ServiceAccount", - metadata: { - name: "ml-pipeline-scheduledworkflow", - namespace: namespace, - }, - }, // service account - - roleBinding:: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "ClusterRoleBinding", - metadata: { - labels: { - app: "ml-pipeline-scheduledworkflow", - }, - name: "ml-pipeline-scheduledworkflow", - }, - roleRef: { - apiGroup: "rbac.authorization.k8s.io", - kind: "ClusterRole", - // TODO: These permissions are too broad. This must be fixed. - name: "cluster-admin", - }, - subjects: [ - { - kind: "ServiceAccount", - name: "ml-pipeline-scheduledworkflow", - namespace: namespace, - }, - ], - }, // role binding - - role: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "Role", - metadata: { - labels: { - app: "ml-pipeline-scheduledworkflow", - }, - name: "ml-pipeline-scheduledworkflow", - namespace: namespace, - }, - rules: [ - { - apiGroups: [ - "argoproj.io", - ], - resources: [ - "workflows", - ], - verbs: [ - "create", - "get", - "list", - "watch", - "update", - "patch", - "delete", - ], - }, - { - apiGroups: [ - "kubeflow.org", - ], - resources: [ - "scheduledworkflows", - ], - verbs: [ - "create", - "get", - "list", - "watch", - "update", - "patch", - "delete", - ], - }, - ], - }, // role - - deploy(image): { - apiVersion: "apps/v1beta2", - kind: "Deployment", - metadata: { - labels: { - app: "ml-pipeline-scheduledworkflow", - }, - name: "ml-pipeline-scheduledworkflow", - namespace: namespace, - }, - spec: { - selector: { - matchLabels: { - app: "ml-pipeline-scheduledworkflow", - }, - }, - template: { - metadata: { - labels: { - app: "ml-pipeline-scheduledworkflow", - }, - }, - spec: { - containers: [ - { - name: "ml-pipeline-scheduledworkflow", - image: image, - imagePullPolicy: "Always", - env: [ - { - name: "POD_NAMESPACE", - valueFrom: { - fieldRef: { - fieldPath: "metadata.namespace", - }, - }, - }, - ], - }, - ], - serviceAccountName: "ml-pipeline-scheduledworkflow", - }, - }, - }, - }, // deploy - crd: { - apiVersion: "apiextensions.k8s.io/v1beta1", - kind: "CustomResourceDefinition", - metadata: { - name: "scheduledworkflows.kubeflow.org", - }, - spec: { - group: "kubeflow.org", - names: { - kind: "ScheduledWorkflow", - listKind: "ScheduledWorkflowList", - plural: "scheduledworkflows", - shortNames: [ - "swf", - ], - singular: "scheduledworkflow", - }, - scope: "Namespaced", - version: "v1beta1", - }, - }, // crd - }, // parts -} diff --git a/pipeline/pipeline/pipeline-ui.libsonnet b/pipeline/pipeline/pipeline-ui.libsonnet deleted file mode 100644 index 61bd54ee196..00000000000 --- a/pipeline/pipeline/pipeline-ui.libsonnet +++ /dev/null @@ -1,189 +0,0 @@ -{ - all(namespace, uiImage):: [ - $.parts(namespace).serviceAccount, - $.parts(namespace).serviceUi, - $.parts(namespace).tensorboardData, - $.parts(namespace).roleBinding, - $.parts(namespace).role, - $.parts(namespace).deployUi(uiImage), - ], - parts(namespace):: { - serviceAccount: { - apiVersion: "v1", - kind: "ServiceAccount", - metadata: { - name: "ml-pipeline-ui", - namespace: namespace, - }, - }, // service account - - serviceUi: { - apiVersion: "v1", - kind: "Service", - metadata: { - labels: { - app: "ml-pipeline-ui", - }, - name: "ml-pipeline-ui", - namespace: namespace, - annotations: { - "getambassador.io/config": - std.join("\n", [ - "---", - "apiVersion: ambassador/v0", - "kind: Mapping", - "name: pipelineui-mapping", - "prefix: /pipeline", - "rewrite: /pipeline", - "timeout_ms: 300000", - "service: ml-pipeline-ui." + namespace, - "use_websocket: true", - ]), - }, //annotations - }, - spec: { - ports: [ - { - port: 80, - targetPort: 3000, - }, - ], - selector: { - app: "ml-pipeline-ui", - }, - }, - status: { - loadBalancer: {}, - }, - }, //serviceUi - - tensorboardData: { - apiVersion: "v1", - kind: "Service", - metadata: { - labels: { - app: "ml-pipeline-tensorboard-ui", - }, - name: "ml-pipeline-tensorboard-ui", - namespace: namespace, - annotations: { - "getambassador.io/config": - std.join("\n", [ - "---", - "apiVersion: ambassador/v0", - "kind: Mapping", - "name: pipeline-tensorboard-ui-mapping", - "prefix: /data", - "rewrite: /data", - "timeout_ms: 300000", - "service: ml-pipeline-ui." + namespace, - "use_websocket: true", - ]), - }, //annotations - }, - spec: { - ports: [ - { - port: 80, - targetPort: 3000, - }, - ], - selector: { - app: "ml-pipeline-tensorboard-ui", - }, - }, - status: { - loadBalancer: {}, - }, - }, //tensorboardData - - roleBinding:: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "RoleBinding", - metadata: { - labels: { - app: "ml-pipeline-ui", - }, - name: "ml-pipeline-ui", - namespace: namespace, - }, - roleRef: { - apiGroup: "rbac.authorization.k8s.io", - kind: "Role", - name: "ml-pipeline-ui", - }, - subjects: [ - { - kind: "ServiceAccount", - name: "ml-pipeline-ui", - namespace: namespace, - }, - ], - }, // role binding - - role: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "Role", - metadata: { - labels: { - app: "ml-pipeline-ui", - }, - name: "ml-pipeline-ui", - namespace: namespace, - }, - rules: [ - { - apiGroups: [""], - resources: [ - "pods", - "pods/log", - ], - verbs: [ - "create", - "get", - "list", - ], - }, - ], - }, // role - - deployUi(image): { - apiVersion: "apps/v1beta2", - kind: "Deployment", - metadata: { - labels: { - app: "ml-pipeline-ui", - }, - name: "ml-pipeline-ui", - namespace: namespace, - }, - spec: { - selector: { - matchLabels: { - app: "ml-pipeline-ui", - }, - }, - template: { - metadata: { - labels: { - app: "ml-pipeline-ui", - }, - }, - spec: { - containers: [ - { - name: "ml-pipeline-ui", - image: image, - imagePullPolicy: "Always", - ports: [{ - containerPort: 3000, - }], - }, - ], - serviceAccountName: "ml-pipeline-ui", - }, - }, - }, - }, // deployUi - }, // parts -} diff --git a/pipeline/pipeline/prototypes/pipeline.jsonnet b/pipeline/pipeline/prototypes/pipeline.jsonnet deleted file mode 100644 index 5dd608172d9..00000000000 --- a/pipeline/pipeline/prototypes/pipeline.jsonnet +++ /dev/null @@ -1,18 +0,0 @@ -// @apiVersion 0.1 -// @name io.ksonnet.pkg.pipeline -// @description ML pipeline. Currently includes pipeline API server, frontend and dependencies. -// @shortDescription ML pipeline -// @param name string Name to give to each of the components -// @optionalParam apiImage string gcr.io/ml-pipeline/api-server:0.1.3 API docker image -// @optionalParam scheduledWorkflowImage string gcr.io/ml-pipeline/scheduledworkflow:0.1.3 schedule workflow docker image -// @optionalParam persistenceAgentImage string gcr.io/ml-pipeline/persistenceagent:0.1.3 persistence agent docker image -// @optionalParam uiImage string gcr.io/ml-pipeline/frontend:0.1.3 UI docker image -// @optionalParam deployArgo string false flag to deploy argo -// @optionalParam reportUsage string false flag to report usage -// @optionalParam mysqlImage string mysql:5.6 mysql image -// @optionalParam minioImage string minio/minio:RELEASE.2018-02-09T22-40-05Z minio image - -local k = import "k.libsonnet"; -local all = import "pipeline/pipeline/all.libsonnet"; - -std.prune(k.core.v1.list.new(all.parts(env, params).all)) diff --git a/pipeline/pipeline/spartakus.libsonnet b/pipeline/pipeline/spartakus.libsonnet deleted file mode 100644 index b7b25412567..00000000000 --- a/pipeline/pipeline/spartakus.libsonnet +++ /dev/null @@ -1,99 +0,0 @@ -{ - all(namespace, usageId):: [ - $.parts(namespace).serviceAccount, - $.parts(namespace).clusterRole, - $.parts(namespace).clusterRoleBinding, - $.parts(namespace).deployVolunteer(usageId), - ], - - parts(namespace):: { - serviceAccount: { - apiVersion: "v1", - kind: "ServiceAccount", - metadata: { - name: "spartakus", - namespace: namespace, - }, - }, // service account - clusterRole: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "ClusterRole", - metadata: { - name: "spartakus", - }, - rules: [ - { - apiGroups: [ - "", - ], - resources: [ - "nodes", - ], - verbs: [ - "get", - "list", - ], - }, - ], - }, // clusterRole - clusterRoleBinding: { - apiVersion: "rbac.authorization.k8s.io/v1beta1", - kind: "ClusterRoleBinding", - metadata: { - name: "spartakus", - }, - roleRef: { - kind: "ClusterRole", - name: "spartakus", - }, - subjects: [ - { - kind: "ServiceAccount", - name: "spartakus", - namespace: namespace, - }, - ], - }, // clusterRoleBinding - deployVolunteer(usageId): { - apiVersion: "apps/v1beta2", - kind: "Deployment", - metadata: { - labels: { - app: "pipeline-spartakus-volunteer", - }, - name: "pipeline-spartakus-volunteer", - namespace: namespace, - }, - spec: { - selector: { - matchLabels: { - app: "pipeline-spartakus-volunteer", - }, - }, - replicas: 1, - template: { - metadata: { - labels: { - app: "pipeline-spartakus-volunteer", - }, - }, - spec: { - containers: [ - { - name: "pipeline-spartakus-volunteer", - image: "gcr.io/google_containers/spartakus-amd64:v1.0.0", - imagePullPolicy: "IfNotPresent", - args: [ - "volunteer", - "--cluster-id=" + usageId, - "--database=https://ml-pipeline-reporting.appspot.com/", - ], - }, - ], - serviceAccountName: "spartakus", - }, - }, - }, - }, // deployVolunteer - }, // parts -} diff --git a/pipeline/registry.yaml b/pipeline/registry.yaml deleted file mode 100644 index ed1caef2394..00000000000 --- a/pipeline/registry.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: '0.1' -kind: ksonnet.io/registry -libraries: - pipeline: - version: master - path: pipeline diff --git a/test/build_image.yaml b/test/build_image.yaml index 21126fb1302..23d02f6ea56 100644 --- a/test/build_image.yaml +++ b/test/build_image.yaml @@ -21,7 +21,6 @@ spec: arguments: parameters: - name: image-build-context-gcs-uri - - name: bootstrapper-image - name: api-image - name: frontend-image - name: scheduledworkflow-image @@ -30,23 +29,12 @@ spec: - name: build-images inputs: parameters: - - name: bootstrapper-image - name: api-image - name: frontend-image - name: scheduledworkflow-image - name: persistenceagent-image steps: - - - name: build-bootstrapper-image - template: build-image - arguments: - parameters: - - name: image-build-context-gcs-uri - value: "{{workflow.parameters.image-build-context-gcs-uri}}" - - name: docker-path - value: pipeline - - name: image-name - value: "{{inputs.parameters.bootstrapper-image}}" - - name: build-api-server-image + - - name: build-api-server-image template: build-image arguments: parameters: diff --git a/test/presubmit-tests.sh b/test/presubmit-tests.sh index 2943dd7bbec..813f4572d12 100755 --- a/test/presubmit-tests.sh +++ b/test/presubmit-tests.sh @@ -127,7 +127,6 @@ ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \ -p target-image-prefix="${GCR_IMAGE_BASE_DIR}/" \ -p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \ -p cluster-type="${CLUSTER_TYPE}" \ --p bootstrapper-image="${GCR_IMAGE_BASE_DIR}/bootstrapper" \ -p api-image="${GCR_IMAGE_BASE_DIR}/api" \ -p frontend-image="${GCR_IMAGE_BASE_DIR}/frontend" \ -p scheduledworkflow-image="${GCR_IMAGE_BASE_DIR}/scheduledworkflow" \ diff --git a/uninstaller.yaml b/uninstaller.yaml deleted file mode 100644 index e9d82feb74e..00000000000 --- a/uninstaller.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: batch/v1 -kind: Job -metadata: - generateName: uninstall-ml-pipeline- -spec: - backoffLimit: 1 - template: - metadata: - name: uninstall-ml-pipeline - spec: - containers: - - name: uninstaller - image: gcr.io/ml-pipeline/bootstrapper:0.1.4 - imagePullPolicy: 'Always' - # Additional parameter available: - args: [ - # "--namespace", "foo", - # "--report_usage", "false", - "--uninstall", - ] - restartPolicy: Never