Build parallel (#85)

* build in parallel containers Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * use different path for each component Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * fix Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * fix Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * fix run-tests Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com>
kubeflow · May 16, 2018 · f140a5b · f140a5b
1 parent 0a95175
commit f140a5b
Show file tree

Hide file tree

Showing 6 changed files with 117 additions and 47 deletions.
diff --git a/scripts/deploy.sh b/scripts/deploy.sh
@@ -28,4 +28,5 @@ kubectl apply -f manifests/modeldb/frontend
 kubectl apply -f manifests/vizier/db
 kubectl apply -f manifests/vizier/core
 kubectl apply -f manifests/vizier/suggestion/random
+kubectl apply -f manifests/vizier/suggestion/grid
 cd - > /dev/null
diff --git a/test/scripts/build-earlystoppings.sh b/test/scripts/build-earlystoppings.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This shell script is used to build an image from our argo workflow
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH}
+REGISTRY="${GCP_REGISTRY}"
+PROJECT="${GCP_PROJECT}"
+GO_DIR=${GOPATH}/src/github.com/${REPO_OWNER}/${REPO_NAME}_earlystoppings
+VERSION=$(git describe --tags --always --dirty)
+
+echo "Activating service-account"
+gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
+
+echo "Copy source to GOPATH"
+mkdir -p ${GO_DIR}
+cp -r cmd ${GO_DIR}/cmd
+cp -r pkg ${GO_DIR}/pkg
+cp -r vendor ${GO_DIR}/vendor
+
+cd ${GO_DIR}
+
+cp cmd/earlystopping/medianstopping/Dockerfile .
+gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/earlystopping-medianstopping:${VERSION} --project=${PROJECT}
diff --git a/test/scripts/build-manager.sh b/test/scripts/build-manager.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This shell script is used to build an image from our argo workflow
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH}
+REGISTRY="${GCP_REGISTRY}"
+PROJECT="${GCP_PROJECT}"
+GO_DIR=${GOPATH}/src/github.com/${REPO_OWNER}/${REPO_NAME}
+VERSION=$(git describe --tags --always --dirty)
+
+echo "Activating service-account"
+gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
+
+echo "Copy source to GOPATH"
+mkdir -p ${GO_DIR}
+cp -r cmd ${GO_DIR}/cmd
+cp -r pkg ${GO_DIR}/pkg
+cp -r vendor ${GO_DIR}/vendor
+cp -r modeldb ${GO_DIR}/modeldb
+
+cd ${GO_DIR}
+cp cmd/manager/Dockerfile .
+gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/vizier-core:${VERSION} --project=${PROJECT}
+
+cp modeldb/Dockerfile .
+gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/katib-frontend:${VERSION} --project=${PROJECT} 
+
+go build -o bin/katib github.com/kubeflow/katib/cmd/cli
diff --git a/test/scripts/build.sh → test/scripts/build-suggestions.sh b/test/scripts/build.sh → test/scripts/build-suggestions.sh
@@ -23,59 +23,28 @@ set -o pipefail
 export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH}
 REGISTRY="${GCP_REGISTRY}"
 PROJECT="${GCP_PROJECT}"
-GO_DIR=${GOPATH}/src/github.com/${REPO_OWNER}/${REPO_NAME}
+GO_DIR=${GOPATH}/src/github.com/${REPO_OWNER}/${REPO_NAME}-suggestions
 VERSION=$(git describe --tags --always --dirty)
 
 echo "Activating service-account"
 gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
 
-echo "Create symlink to GOPATH"
-mkdir -p ${GOPATH}/src/github.com/${REPO_OWNER}
-ln -s ${PWD} ${GO_DIR}
+echo "Copy source to GOPATH"
+mkdir -p ${GO_DIR}
+cp -r cmd ${GO_DIR}/cmd
+cp -r pkg ${GO_DIR}/pkg
+cp -r vendor ${GO_DIR}/vendor
 
 cd ${GO_DIR}
-#echo "building container in gcloud"
-#gcloud version
-# gcloud components update -q
-
-pids=()
-cp cmd/manager/Dockerfile .
-gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/vizier-core:${VERSION} --project=${PROJECT} &
-pids+=($!)
-sleep 30 # wait for copy code to gcloud
 
 cp cmd/suggestion/random/Dockerfile .
-gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-random:${VERSION} --project=${PROJECT} &
-pids+=($!)
-sleep 30 # wait for copy code to gcloud
+gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-random:${VERSION} --project=${PROJECT}
 
 cp cmd/suggestion/grid/Dockerfile .
-gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-grid:${VERSION} --project=${PROJECT} &
-pids+=($!)
-sleep 30 # wait for copy code to gcloud
+gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-grid:${VERSION} --project=${PROJECT}
 
 #cp cmd/suggestion/hyperband/Dockerfile .
-#gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-hyperband:${VERSION} --project=${PROJECT} &
-#pids+=($!)
-#sleep 30 # wait for copy code to gcloud
+#gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-hyperband:${VERSION} --project=${PROJECT}
 
 cp cmd/suggestion/bayesianoptimization/Dockerfile .
-gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-bayesianoptimization:${VERSION} --project=${PROJECT} &
-pids+=($!)
-sleep 30 # wait for copy code to gcloud
-
-cp cmd/earlystopping/medianstopping/Dockerfile .
-gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/earlystopping-medianstopping:${VERSION} --project=${PROJECT} &
-pids+=($!)
-sleep 30 # wait for copy code to gcloud
-
-cp modeldb/Dockerfile .
-gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/katib-frontend:${VERSION} --project=${PROJECT} &
-pids+=($!)
-
-for pid in ${pids[@]}; do
-  wait $pid
-  if [ $? -ne 0 ]; then
-    exit 1
-  fi
-done
+gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-bayesianoptimization:${VERSION} --project=${PROJECT}
diff --git a/test/scripts/run-tests.sh b/test/scripts/run-tests.sh
@@ -49,7 +49,6 @@ sed -i -e "s@image: katib\/suggestion-grid@image: ${REGISTRY}\/${REPO_NAME}\/sug
 #sed -i -e "s@image: katib\/suggestion-bayesianoptimization@image: ${REGISTRY}\/${REPO_NAME}\/suggestion-bayesianoptimization:${VERSION}@" manifests/vizier/suggestion/bayesianoptimization/deployment.yaml
 sed -i -e "s@image: katib\/earlystopping-medianstopping@image: ${REGISTRY}\/${REPO_NAME}\/earlystopping-medianstopping:${VERSION}@" manifests/vizier/earlystopping/medianstopping/deployment.yaml
 sed -i -e "s@image: katib\/katib-frontend@image: ${REGISTRY}\/${REPO_NAME}\/katib-frontend:${VERSION}@" manifests/modeldb/frontend/deployment.yaml
-cd ${GO_DIR}
 ./scripts/deploy.sh
 
 TIMEOUT=60

diff --git a/test/workflows/components/workflows.libsonnet b/test/workflows/components/workflows.libsonnet
@@ -203,8 +203,16 @@
                 }],
                 [
                   {
-                    name: "build",
-                    template: "build",
+                    name: "build-manager",
+                    template: "build-manager",
+                  },
+                  {
+                    name: "build-suggestions",
+                    template: "build-suggestions",
+                  },
+                  {
+                    name: "build-earlystoppings",
+                    template: "build-earlystoppings",
                   },
                   {
                     name: "create-pr-symlink",
@@ -291,9 +299,15 @@
               "copy_artifacts",
               "--bucket=" + bucket,
             ]),  // copy-artifacts
-            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("build", testWorkerImage, [
-              "test/scripts/build.sh",
-            ]),  // build
+            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("build-manager", testWorkerImage, [
+              "test/scripts/build-manager.sh",
+            ]),  // build-manager
+            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("build-suggestions", testWorkerImage, [
+              "test/scripts/build-suggestions.sh",
+            ]),  // build-suggestions
+            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("build-earlystoppings", testWorkerImage, [
+              "test/scripts/build-earlystoppings.sh",
+            ]),  // build-earlystoppings
             $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("unit-test", testWorkerImage, [
               "test/scripts/unit-test.sh",
             ]),  // unit test