Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(backend): Upgrade Argo server image version to v2.11.6 #4693

Closed
wants to merge 7 commits into from
Closed

feat(backend): Upgrade Argo server image version to v2.11.6 #4693

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
538e167
Upgrade Argo workflow controller manifest
xinbinhuang Oct 30, 2020
94b2947
fixup! Upgrade Argo workflow controller manifest
xinbinhuang Oct 30, 2020
36684b1
Upgrade argo docker base image version
xinbinhuang Oct 30, 2020
c49b70c
fixup! Upgrade argo docker base image version
xinbinhuang Oct 30, 2020
c4998bf
fixup! fixup! Upgrade argo docker base image version
xinbinhuang Oct 30, 2020
80db306
fixup! fixup! fixup! Upgrade argo docker base image version
xinbinhuang Oct 30, 2020
44e41e6
Remove deprecated executorImage key from configmap
xinbinhuang Nov 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .cloudbuild.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,10 +224,10 @@ steps:
args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.14']
id: 'pullCloudsqlProxy'
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance']
args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.11.6-license-compliance']
id: 'pullArgoExecutor'
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance']
args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.11.6-license-compliance']
id: 'pullArgoWorkflowController'

# Tag for Hosted - SemVersion to Major.Minor parsing
Expand Down
20 changes: 10 additions & 10 deletions .release.cloudbuild.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,14 +485,14 @@ steps:
docker push gcr.io/ml-pipeline/google/pipelines-test/cloudsqlproxy:$(cat /workspace/mm.ver)

- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance']
args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.11.6-license-compliance']
id: 'pullArgoExecutor'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME']
args: ['tag', 'gcr.io/ml-pipeline/argoexec:v2.11.6-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME']
id: 'tagArgoExecutorForMarketplace'
waitFor: ['pullArgoExecutor']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$TAG_NAME']
args: ['tag', 'gcr.io/ml-pipeline/argoexec:v2.11.6-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$TAG_NAME']
id: 'tagArgoExecutorForMarketplaceTest'
waitFor: ['pullArgoExecutor']
- id: 'tagArgoExecutorForMarketplaceMajorMinor'
Expand All @@ -502,20 +502,20 @@ steps:
args:
- -ceux
- |
docker tag gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
docker tag gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
docker tag gcr.io/ml-pipeline/argoexec:v2.11.6-license-compliance gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
docker tag gcr.io/ml-pipeline/argoexec:v2.11.6-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)

- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance']
args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.11.6-license-compliance']
id: 'pullArgoWorkflowController'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME']
args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v2.11.6-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME']
id: 'tagArgoWorkflowControllerForMarketplace'
waitFor: ['pullArgoWorkflowController']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$TAG_NAME']
args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v2.11.6-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$TAG_NAME']
id: 'tagArgoWorkflowControllerForMarketplaceTest'
waitFor: ['pullArgoWorkflowController']
- id: 'tagArgoWorkflowControllerForMarketplaceMajorMinor'
Expand All @@ -525,8 +525,8 @@ steps:
args:
- -ceux
- |
docker tag gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
docker tag gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
docker tag gcr.io/ml-pipeline/workflow-controller:v2.11.6-license-compliance gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
docker tag gcr.io/ml-pipeline/workflow-controller:v2.11.6-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)

Expand Down
2 changes: 1 addition & 1 deletion backend/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ COPY backend/requirements.txt .
RUN python3 -m pip install -r requirements.txt

# Downloading Argo CLI so that the samples are validated
#ADD https://github.com/argoproj/argo/releases/download/v2.7.5/argo-linux-amd64 /usr/local/bin/argo
#ADD https://github.com/argoproj/argo/releases/download/v2.11.6/argo-linux-amd64 /usr/local/bin/argo
Copy link
Author

@xinbinhuang xinbinhuang Oct 30, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I replace all v2.7.5 -> v2.11.6 in the repo, but I am not sure if I need to update this one.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should leave this one as well since it is the Kubeflow backend which will be running 2.7.7. Argo is in this case used to validate the workflows to check if they are valid according to argo.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems that we have several places that use Argo but they can be somehow independent of each other. Can you tell me how each of them is used so that I can get a clearer picture of how they work together?

ADD https://github.com/argoproj/argo/releases/download/v2.4.3/argo-linux-amd64 /usr/local/bin/argo
RUN chmod +x /usr/local/bin/argo

Expand Down
2 changes: 1 addition & 1 deletion manifests/kustomize/base/argo/workflow-controller-configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ data:
config: |
{
namespace: $(kfp-namespace),
executorImage: gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance,
executorImage: gcr.io/ml-pipeline/argoexec:v2.11.6-license-compliance,
containerRuntimeExecutor: $(kfp-container-runtime-executor),
artifactRepository:
{
Expand Down
5 changes: 3 additions & 2 deletions manifests/kustomize/base/argo/workflow-controller-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ spec:
- --configmap
- workflow-controller-configmap
- --executor-image
- gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance
- gcr.io/ml-pipeline/argoexec:v2.11.6-license-compliance
- --namespaced
Copy link
Author

@xinbinhuang xinbinhuang Oct 30, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the extra argument that I need to specifically add for my deployment.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a corresponding documentation? Does omitting the argument make it cluster mode? We need to make sure it is customizable from an overlay to continue support multi-user mode.

Copy link
Author

@xinbinhuang xinbinhuang Nov 7, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The cluster vs namespace install seems to start since v2.5 - this can also be found in the v2.5 release branch.

In my case, it did become a cluster install if I simply replace the image version to argoproj/argoexec:v2.11.6 - I got permission issues when trying to launch new pipelines.

I am not familiar with the multi-user mode. Does it install the workflow-controller in cluster scope?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Error log from when trying to launch pipelines without --namespaced

E1107 01:14:06.943364       1 reflector.go:153] pkg/mod/k8s.io/client-go@v0.17.8/tools/cache/reflector.go:105: Failed to list *unstructured.Unstructured: workflowtemplates.argoproj.io is forbidden: User "system:serviceaccount:kubeflow:argo" cannot list resource "workflowtemplates" in API group "argoproj.io" at the cluster scope

command:
- workflow-controller
env:
Expand All @@ -36,7 +37,7 @@ spec:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
image: gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance
image: gcr.io/ml-pipeline/workflow-controller:v2.11.6-license-compliance
imagePullPolicy: IfNotPresent
name: workflow-controller
resources: {}
Expand Down
2 changes: 1 addition & 1 deletion manifests/kustomize/env/aws/config
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
namespace: $(kfp-namespace),
executorImage: gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance,
executorImage: gcr.io/ml-pipeline/argoexec:v2.11.6-license-compliance,
containerRuntimeExecutor: $(kfp-container-runtime-executor),
artifactRepository:
{
Expand Down
2 changes: 1 addition & 1 deletion test/install-argo.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ ACCOUNT=$(gcloud info --format='value(config.account)')
kubectl create clusterrolebinding PROW_BINDING --clusterrole=cluster-admin --user=$ACCOUNT --dry-run -o yaml | kubectl apply -f -
kubectl create clusterrolebinding DEFAULT_BINDING --clusterrole=cluster-admin --serviceaccount=default:default --dry-run -o yaml | kubectl apply -f -

ARGO_VERSION=v2.7.5
ARGO_VERSION=v2.11.6
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if I need to update this one

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this will be needed for all the test infrastructure to run with argo 2.11.6


# if argo is not installed
if ! which argo; then
Expand Down
2 changes: 1 addition & 1 deletion test/sample-test/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ RUN cd /python/src/github.com/kubeflow/pipelines
# Install argo.
RUN echo "install argo"
RUN mkdir -p /usr/local/bin/
#RUN ARGO_VERSION=v2.7.5 && curl -sSL -o /usr/local/bin/argo \
#RUN ARGO_VERSION=v2.11.6 && curl -sSL -o /usr/local/bin/argo \
RUN ARGO_VERSION=v2.4.3 && curl -sSL -o /usr/local/bin/argo \
Comment on lines +25 to 26
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure about this one either.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is used to kick of the ML pipeline sample e2e test, since the changes you do is only for the server side I think we should not update this to 2.11.6. But I think we should go with 2.7.7 since that is the version that is used by Kubeflow today.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There was a bug from argo that prevented using the argo cli from a container, that's why we were stuck on 2.4.3. I'm not sure about latest status, if the latest version works, we can upgrade too.
/cc @Ark-kun can you provide context about that issue?

"https://github.com/argoproj/argo/releases/download/$ARGO_VERSION/argo-linux-amd64" && \
chmod +x /usr/local/bin/argo
Expand Down
8 changes: 4 additions & 4 deletions test/tag_for_hosted.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,12 +118,12 @@ docker tag gcr.io/cloudsql-docker/gce-proxy:1.14 gcr.io/$PROJECT_ID/hosted/$COMM
docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cloudsqlproxy:$SEM_VER
docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cloudsqlproxy:$MM_VER

docker tag gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$SEM_VER
docker tag gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$MM_VER
docker tag gcr.io/ml-pipeline/argoexec:v2.11.6-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$SEM_VER
docker tag gcr.io/ml-pipeline/argoexec:v2.11.6-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$MM_VER
docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$SEM_VER
docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$MM_VER

docker tag gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$SEM_VER
docker tag gcr.io/ml-pipeline/workflow-controller:v2.7.5-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$MM_VER
docker tag gcr.io/ml-pipeline/workflow-controller:v2.11.6-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$SEM_VER
docker tag gcr.io/ml-pipeline/workflow-controller:v2.11.6-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$MM_VER
docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$SEM_VER
docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$MM_VER
2 changes: 1 addition & 1 deletion third_party/argo/Dockerfile.argoexec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ RUN sh /third_party/download_source.sh </third_party/argo/repo-MPL.txt


# Argo exec image
From argoproj/argoexec:v2.7.5
From argoproj/argoexec:v2.11.6

# Copy concatenated license file
COPY third_party/argo/license.txt /third_party/license.txt
Expand Down
2 changes: 1 addition & 1 deletion third_party/argo/Dockerfile.workflow-controller
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ RUN apk add --no-cache curl
ENV OUTPUT_FOLDER /downloads/
RUN sh /third_party/download_source.sh </third_party/argo/repo-MPL.txt

From argoproj/workflow-controller:v2.7.5
From argoproj/workflow-controller:v2.11.6

# Copy concatenated license file
COPY third_party/argo/license.txt /third_party/license.txt
Expand Down
2 changes: 1 addition & 1 deletion third_party/argo/release.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ set -e
# $PROJECT: gcp project

RELEASE_PROJECT=ml-pipeline
TAG=v2.7.5-license-compliance
TAG=v2.11.6-license-compliance

gcloud builds submit --config third_party/argo/cloudbuild.yaml . \
--substitutions=TAG_NAME="$TAG" --project "$PROJECT"
Expand Down