Build all images from source code prepared by Prow. Got rid of git pulls

All images are now built from archived version of code the source code prepared by Prow. This saves 25 more pulls and improves test reliability. The archived source code location is passed through image-build-context-gcs-uri parameter.
kubeflow · Nov 23, 2018 · 6e44c03 · 6e44c03
1 parent 9e59ecb
commit 6e44c03
Show file tree

Hide file tree

Showing 7 changed files with 98 additions and 96 deletions.
diff --git a/test/build_image.yaml b/test/build_image.yaml
@@ -20,8 +20,7 @@ spec:
   entrypoint: build-images
   arguments:
     parameters:
-    - name: commit-sha
-      value: master
+    - name: image-build-context-gcs-uri
     - name: bootstrapper-image
     - name: api-image
     - name: frontend-image
@@ -31,7 +30,6 @@ spec:
   - name: build-images
     inputs:
       parameters:
-      - name: commit-sha
       - name: bootstrapper-image
       - name: api-image
       - name: frontend-image
@@ -42,8 +40,8 @@ spec:
         template: build-image
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
+          - name: image-build-context-gcs-uri
+            value: "{{workflow.parameters.image-build-context-gcs-uri}}"
           - name: docker-path
             value: pipeline
           - name: image-name
@@ -52,8 +50,8 @@ spec:
         template: build-image
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
+          - name: image-build-context-gcs-uri
+            value: "{{workflow.parameters.image-build-context-gcs-uri}}"
           - name: docker-path
             value: .
           - name: image-name
@@ -64,8 +62,8 @@ spec:
         template: build-image
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
+          - name: image-build-context-gcs-uri
+            value: "{{workflow.parameters.image-build-context-gcs-uri}}"
           - name: docker-path
             value: .
           - name: docker-file
@@ -76,8 +74,8 @@ spec:
         template: build-image
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
+          - name: image-build-context-gcs-uri
+            value: "{{workflow.parameters.image-build-context-gcs-uri}}"
           - name: docker-path
             value: .
           - name: image-name
@@ -88,8 +86,8 @@ spec:
         template: build-image
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
+          - name: image-build-context-gcs-uri
+            value: "{{workflow.parameters.image-build-context-gcs-uri}}"
           - name: docker-path
             value: .
           - name: image-name
@@ -101,19 +99,24 @@ spec:
   - name: build-image
     inputs:
       parameters:
-      # The github commit-sha to pull code from
-      - name: commit-sha
+      # GCS URI prefix pointing to a .tar.gz archive of Docker build context
+      - name: image-build-context-gcs-uri
       # The relative code path to the Dockerfile
       - name: docker-path
       # Name of the Docker file to use. "Dockerfile" by default
       - name: docker-file
         value: Dockerfile
       - name: image-name
+    outputs:
+      parameters:
+        - name: strict-image-name
+          valueFrom:
+            path: /outputs/strict-image-name/file
     container:
-      image: gcr.io/ml-pipeline-staging/image-builder:v20181108-0.1.1-39-g2274e35-e3b0c4
+      image: gcr.io/ml-pipeline-staging/image-builder:v20181120-0.1.3-rc.1-21-gb7c8af0-dirty-9786df
       imagePullPolicy: 'Always'
       args: [
-        "--commit_sha", "{{inputs.parameters.commit-sha}}",
+        "--image-build-context-gcs-uri", "{{inputs.parameters.image-build-context-gcs-uri}}",
         "--docker_path", "{{inputs.parameters.docker-path}}",
         "--docker_file", "{{inputs.parameters.docker-file}}",
         "--image_name", "{{inputs.parameters.image-name}}",

diff --git a/test/e2e_test_gke.yaml b/test/e2e_test_gke.yaml
@@ -20,7 +20,7 @@ spec:
   entrypoint: integration-test
   arguments:
     parameters:
-    - name: commit-sha
+    - name: image-build-context-gcs-uri
     - name: target-image-prefix
     - name: test-results-gcs-dir
     - name: bootstrapper-image
@@ -42,7 +42,6 @@ spec:
   - name: integration-test
     inputs:
       parameters:
-      - name: commit-sha
       - name: target-image-prefix
       - name: test-results-gcs-dir
       - name: bootstrapper-image
@@ -59,8 +58,6 @@ spec:
         template: build-image
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
           - name: docker-path
             value: .
           - name: docker-file
@@ -71,8 +68,6 @@ spec:
         template: build-image
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
           - name: docker-path
             value: test/frontend-integration-test
           - name: image-name
@@ -81,8 +76,6 @@ spec:
         template: build-image
         arguments:
           parameters:
-          - name: commit-sha
-            value: "{{inputs.parameters.commit-sha}}"
           - name: docker-path
             value: .
           - name: docker-file
@@ -187,19 +180,25 @@ spec:
   - name: build-image
     inputs:
       parameters:
-      # The github commit-sha to pull code from
-      - name: commit-sha
+      # GCS URI prefix pointing to a .tar.gz archive of Docker build context
+      - name: image-build-context-gcs-uri
+        value: "{{workflow.parameters.image-build-context-gcs-uri}}"
       # The relative code path to the Dockerfile
       - name: docker-path
       # Name of the Docker file to use. "Dockerfile" by default
       - name: docker-file
         value: Dockerfile
       - name: image-name
+    outputs:
+      parameters:
+        - name: strict-image-name
+          valueFrom:
+            path: /outputs/strict-image-name/file
     container:
-      image: gcr.io/ml-pipeline-staging/image-builder:v20181108-0.1.1-39-g2274e35-e3b0c4
+      image: gcr.io/ml-pipeline-staging/image-builder:v20181120-0.1.3-rc.1-21-gb7c8af0-dirty-9786df
       imagePullPolicy: 'Always'
       args: [
-        "--commit_sha", "{{inputs.parameters.commit-sha}}",
+        "--image-build-context-gcs-uri", "{{inputs.parameters.image-build-context-gcs-uri}}",
         "--docker_path", "{{inputs.parameters.docker-path}}",
         "--docker_file", "{{inputs.parameters.docker-file}}",
         "--image_name", "{{inputs.parameters.image-name}}",

diff --git a/test/imagebuilder/build.sh b/test/imagebuilder/build.sh
@@ -21,7 +21,7 @@ DOCKER_FILE=Dockerfile
 usage()
 {
     echo "usage: deploy.sh
-    [--commit_sha   commit SHA to pull code from]
+    [--image-build-context-gcs-uri   GCS URI pointing to a .tar.gz archive of Docker build context]
     [--docker_path  path to the Dockerfile]
     [--docker_file  name of the Docker file. Dockerfile by default]
     [--image_name   project of the GCR to upload image to]
@@ -31,16 +31,16 @@ usage()
 
 while [ "$1" != "" ]; do
     case $1 in
-             --commit_sha )     shift
-                                COMMIT_SHA=$1
+             --image-build-context-gcs-uri ) shift
+                                CONTEXT_GCS_URI=$1
                                 ;;
              --docker_path )    shift
                                 DOCKER_PATH=$1
                                 ;;
              --docker_file )    shift
                                 DOCKER_FILE=$1
                                 ;;
-             --image_name )     shift
+             --image_name )   shift
                                 IMAGE_NAME=$1
                                 ;;
              --build_script )   shift
@@ -55,28 +55,33 @@ while [ "$1" != "" ]; do
     shift
 done
 
-BASE_DIR=/ml
+BASE_DIR=/image_builder
+mkdir $BASE_DIR
+cd $BASE_DIR
 
-echo "Clone ML pipeline code in COMMIT SHA ${COMMIT_SHA}..."
-git clone https://github.com/kubeflow/pipelines ${BASE_DIR}
-cd ${BASE_DIR}
-git config --local user.name 'K8S Bootstrap'
-git config --local user.email k8s_bootstrap@localhost
-git merge --no-ff ${COMMIT_SHA} -m "Merged PR ${COMMIT_SHA}"
+echo "Downloading Docker build context from $CONTEXT_GCS_URI..."
+downloaded_code_archive_file=$(mktemp)
+gsutil cp "$CONTEXT_GCS_URI" "$downloaded_code_archive_file"
+tar -xzf "$downloaded_code_archive_file" --directory .
 
-echo "Waiting for dind to start..."
+echo "Waiting for Docker-in-Docker daemon to start..."
 until docker ps; do sleep 3; done;
 
+gcloud auth configure-docker
+
 if [ "$BUILD_SCRIPT" == "" ]; then
   echo "Build image ${IMAGE_NAME} using ${BASE_DIR}/${DOCKER_PATH}/${DOCKER_FILE}..."
-  docker build -t ${IMAGE_NAME} -f ${BASE_DIR}/${DOCKER_PATH}/${DOCKER_FILE} ${BASE_DIR}/${DOCKER_PATH}
+  docker build -t ${IMAGE_NAME} -f ${BASE_DIR}/${DOCKER_PATH}/${DOCKER_FILE} ${DOCKER_PATH}
 else
   echo "Build image ${IMAGE_NAME} using ${BUILD_SCRIPT}..."
   cd $(dirname ${BUILD_SCRIPT})
-  gcloud auth configure-docker
   bash $(basename ${BUILD_SCRIPT}) -i ${IMAGE_NAME}
 fi
 
-echo "Push image ${IMAGE_NAME} to gcr..."
-gcloud auth configure-docker
+echo "Pushing image ${IMAGE_NAME}..."
 docker push ${IMAGE_NAME}
+
+image_name_with_digest=$(docker inspect --format="{{index .RepoDigests 0}}" "$IMAGE_NAME")
+strict_image_name_output_file=/outputs/strict-image-name/file
+mkdir -p "$(dirname "$strict_image_name_output_file")"
+echo $image_name_with_digest > "$strict_image_name_output_file"
diff --git a/test/presubmit-tests-gce-minikube.sh b/test/presubmit-tests-gce-minikube.sh
@@ -82,7 +82,7 @@ gcloud compute scp --zone=$ZONE --verbosity=error --recurse "$git_root" $instanc
 gcloud compute ssh --zone=$ZONE $instance_name -- "~/pipelines/test/minikube/install_docker_minikube_argo.sh"
 
 #Running the presubmit tests
-gcloud compute ssh --zone=$ZONE $instance_name -- PULL_PULL_SHA="$PULL_PULL_SHA" WORKSPACE="~/${WORKSPACE}" GOOGLE_APPLICATION_CREDENTIALS="~/service-account.json" "~/pipelines/test/presubmit-tests.sh" --cluster-type none "$@"
+gcloud compute ssh --zone=$ZONE $instance_name -- PULL_PULL_SHA="$PULL_PULL_SHA" PULL_BASE_SHA="$PULL_BASE_SHA" WORKSPACE="~/${WORKSPACE}" GOOGLE_APPLICATION_CREDENTIALS="~/service-account.json" "~/pipelines/test/presubmit-tests.sh" --cluster-type none "$@"
 
 #Copy back the artifacts
 mkdir -p "${ARTIFACT_DIR}"

diff --git a/test/presubmit-tests.sh b/test/presubmit-tests.sh
@@ -67,9 +67,21 @@ PULL_ARGO_WORKFLOW_STATUS_MAX_ATTEMPT=$(expr $TIMEOUT_SECONDS / 20 )
 
 echo "presubmit test starts"
 
+repo_root=$(dirname "$0")/..
+cd "$repo_root"
+
 # activating the service account
 gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
 
+#Uploading the source code to GCS:
+local_code_archive_file=$(mktemp)
+date_string=$(TZ=PST8PDT date +%Y-%m-%d_%H-%M-%S_%Z)
+code_archive_prefix="gs://${TEST_RESULT_BUCKET}/${PULL_PULL_SHA}/source_code"
+remote_code_archive_uri="${code_archive_prefix}_${PULL_BASE_SHA}_${date_string}.tar.gz"
+
+tar -czf "$local_code_archive_file" .
+gsutil cp "$local_code_archive_file" "$remote_code_archive_uri"
+
 #Creating a new GKE cluster if needed
 if [ "$CLUSTER_TYPE" == "create-gke" ]; then
   echo "create test cluster"
@@ -116,7 +128,7 @@ kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo/$ARGO_V
 
 echo "submitting argo workflow for commit ${PULL_PULL_SHA}..."
 ARGO_WORKFLOW=`argo submit $(dirname $0)/${WORKFLOW_FILE} \
--p commit-sha="${PULL_PULL_SHA}" \
+-p image-build-context-gcs-uri="$remote_code_archive_uri" \
 -p target-image-prefix="${GCR_IMAGE_BASE_DIR}/" \
 -p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \
 -p cluster-type="${CLUSTER_TYPE}" \