Skip to content

Commit

Permalink
common build image script (#815)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hongye-sun @IronPan
hongye-sun authored and IronPan committed Feb 13, 2019
1 parent 7c633a9 commit fdfd25f
Show file tree
Hide file tree
Showing 14 changed files with 72 additions and 481 deletions.
38 changes: 1 addition & 37 deletions components/bigquery/query/build_image.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,45 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.

while getopts ":hp:t:i:" opt; do
case "${opt}" in
h) echo "-p: project name"
echo "-t: tag name"
echo "-i: image name. If provided, project name and tag name are not necessary"
exit
;;
p) PROJECT_ID=${OPTARG}
;;
t) TAG_NAME=${OPTARG}
;;
i) IMAGE_NAME=${OPTARG}
;;
\? ) echo "Usage: cmd [-p] project [-t] tag [-i] image"
exit
;;
esac
done

LOCAL_IMAGE_NAME=ml-pipeline-bigquery-query

if [ -z "${PROJECT_ID}" ]; then
PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)")
fi

if [ -z "${TAG_NAME}" ]; then
TAG_NAME=$(date +v%Y%m%d)-$(git describe --tags --always --dirty)-$(git diff | shasum -a256 | cut -c -6)
fi

# build base image
pushd ../base
./build_image.sh
popd

docker build -t ${LOCAL_IMAGE_NAME} .
if [ -z "${IMAGE_NAME}" ]; then
docker tag ${LOCAL_IMAGE_NAME} gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
docker push gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
else
docker tag ${LOCAL_IMAGE_NAME} "${IMAGE_NAME}"
docker push "${IMAGE_NAME}"
fi
../../build_image.sh -l ml-pipeline-bigquery-query "$@"
59 changes: 59 additions & 0 deletions components/build_image.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/bin/bash -e
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

while getopts ":hp:t:i:l:" opt; do
case "${opt}" in
h) echo "-p: project name"
echo "-t: tag name"
echo "-i: image name. If provided, project name and tag name are not necessary"
echo "-l: local image name."
exit
;;
p) PROJECT_ID=${OPTARG}
;;
t) TAG_NAME=${OPTARG}
;;
i) IMAGE_NAME=${OPTARG}
;;
l) LOCAL_IMAGE_NAME=${OPTARG}
;;
\? ) echo "Usage: cmd [-p] project [-t] tag [-i] image [-l] local image"
exit
;;
esac
done

if [ -z "${PROJECT_ID}" ]; then
PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)")
fi

if [ -z "${TAG_NAME}" ]; then
TAG_NAME=$(date +v%Y%m%d)-$(git describe --tags --always --dirty)-$(git diff | shasum -a256 | cut -c -6)
fi

if [ -z "${IMAGE_NAME}" ]; then
docker pull gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:latest || true
fi

docker build -t ${LOCAL_IMAGE_NAME} . --cache-from gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:latest
if [ -z "${IMAGE_NAME}" ]; then
docker tag ${LOCAL_IMAGE_NAME} gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
docker tag ${LOCAL_IMAGE_NAME} gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:latest
docker push gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
docker push gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:latest
else
docker tag ${LOCAL_IMAGE_NAME} "${IMAGE_NAME}"
docker push "${IMAGE_NAME}"
fi
38 changes: 1 addition & 37 deletions components/dataflow/predict/build_image.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,45 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.

while getopts ":hp:t:i:" opt; do
case "${opt}" in
h) echo "-p: project name"
echo "-t: tag name"
echo "-i: image name. If provided, project name and tag name are not necessary"
exit
;;
p) PROJECT_ID=${OPTARG}
;;
t) TAG_NAME=${OPTARG}
;;
i) IMAGE_NAME=${OPTARG}
;;
\? ) echo "Usage: cmd [-p] project [-t] tag [-i] image"
exit
;;
esac
done

LOCAL_IMAGE_NAME=ml-pipeline-dataflow-tf-predict

if [ -z "${PROJECT_ID}" ]; then
PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)")
fi

if [ -z "${TAG_NAME}" ]; then
TAG_NAME=$(date +v%Y%m%d)-$(git describe --tags --always --dirty)-$(git diff | shasum -a256 | cut -c -6)
fi

# build base image
pushd ../base
./build_image.sh
popd

docker build -t ${LOCAL_IMAGE_NAME} .
if [ -z "${IMAGE_NAME}" ]; then
docker tag ${LOCAL_IMAGE_NAME} gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
docker push gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
else
docker tag ${LOCAL_IMAGE_NAME} ${IMAGE_NAME}
docker push ${IMAGE_NAME}
fi
../../build_image.sh -l ml-pipeline-dataflow-tf-predict "$@"
38 changes: 1 addition & 37 deletions components/dataflow/tfdv/build_image.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,45 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.

while getopts ":hp:t:i:" opt; do
case "${opt}" in
h) echo "-p: project name"
echo "-t: tag name"
echo "-i: image name. If provided, project name and tag name are not necessary"
exit
;;
p) PROJECT_ID=${OPTARG}
;;
t) TAG_NAME=${OPTARG}
;;
i) IMAGE_NAME=${OPTARG}
;;
\? ) echo "Usage: cmd [-p] project [-t] tag [-i] image"
exit
;;
esac
done

LOCAL_IMAGE_NAME=ml-pipeline-dataflow-tfdv

if [ -z "${PROJECT_ID}" ]; then
PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)")
fi

if [ -z "${TAG_NAME}" ]; then
TAG_NAME=$(date +v%Y%m%d)-$(git describe --tags --always --dirty)-$(git diff | shasum -a256 | cut -c -6)
fi

# build base image
pushd ../base
./build_image.sh
popd

docker build -t ${LOCAL_IMAGE_NAME} .
if [ -z "${IMAGE_NAME}" ]; then
docker tag ${LOCAL_IMAGE_NAME} gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
docker push gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
else
docker tag ${LOCAL_IMAGE_NAME} "${IMAGE_NAME}"
docker push "${IMAGE_NAME}"
fi
../../build_image.sh -l ml-pipeline-dataflow-tfdv "$@"
38 changes: 1 addition & 37 deletions components/dataflow/tfma/build_image.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,45 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.

while getopts ":hp:t:i:" opt; do
case "${opt}" in
h) echo "-p: project name"
echo "-t: tag name"
echo "-i: image name. If provided, project name and tag name are not necessary"
exit
;;
p) PROJECT_ID=${OPTARG}
;;
t) TAG_NAME=${OPTARG}
;;
i) IMAGE_NAME=${OPTARG}
;;
\? ) echo "Usage: cmd [-p] project [-t] tag [-i] image"
exit
;;
esac
done

LOCAL_IMAGE_NAME=ml-pipeline-dataflow-tfma

if [ -z "${PROJECT_ID}" ]; then
PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)")
fi

if [ -z "${TAG_NAME}" ]; then
TAG_NAME=$(date +v%Y%m%d)-$(git describe --tags --always --dirty)-$(git diff | shasum -a256 | cut -c -6)
fi

# build base image
pushd ../base
./build_image.sh
popd

docker build -t ${LOCAL_IMAGE_NAME} .
if [ -z "${IMAGE_NAME}" ]; then
docker tag ${LOCAL_IMAGE_NAME} gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
docker push gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
else
docker tag ${LOCAL_IMAGE_NAME} "${IMAGE_NAME}"
docker push "${IMAGE_NAME}"
fi
../../build_image.sh -l ml-pipeline-dataflow-tfma "$@"
38 changes: 1 addition & 37 deletions components/dataflow/tft/build_image.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,45 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.

while getopts ":hp:t:i:" opt; do
case "${opt}" in
h) echo "-p: project name"
echo "-t: tag name"
echo "-i: image name. If provided, project name and tag name are not necessary"
exit
;;
p) PROJECT_ID=${OPTARG}
;;
t) TAG_NAME=${OPTARG}
;;
i) IMAGE_NAME=${OPTARG}
;;
\? ) echo "Usage: cmd [-p] project [-t] tag [-i] image"
exit
;;
esac
done

LOCAL_IMAGE_NAME=ml-pipeline-dataflow-tft

if [ -z "${PROJECT_ID}" ]; then
PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)")
fi

if [ -z "${TAG_NAME}" ]; then
TAG_NAME=$(date +v%Y%m%d)-$(git describe --tags --always --dirty)-$(git diff | shasum -a256 | cut -c -6)
fi

# build base image
pushd ../base
./build_image.sh
popd

docker build -t ${LOCAL_IMAGE_NAME} .
if [ -z "${IMAGE_NAME}" ]; then
docker tag ${LOCAL_IMAGE_NAME} gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
docker push gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
else
docker tag ${LOCAL_IMAGE_NAME} "${IMAGE_NAME}"
docker push "${IMAGE_NAME}"
fi
../../build_image.sh -l ml-pipeline-dataflow-tft "$@"
38 changes: 1 addition & 37 deletions components/dataproc/analyze/build_image.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,45 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.

while getopts ":hp:t:i:" opt; do
case "${opt}" in
h) echo "-p: project name"
echo "-t: tag name"
echo "-i: image name. If provided, project name and tag name are not necessary"
exit
;;
p) PROJECT_ID=${OPTARG}
;;
t) TAG_NAME=${OPTARG}
;;
i) IMAGE_NAME=${OPTARG}
;;
\? ) echo "Usage: cmd [-p] project [-t] tag [-i] image"
exit
;;
esac
done

LOCAL_IMAGE_NAME=ml-pipeline-dataproc-analyze

if [ -z "${PROJECT_ID}" ]; then
PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)")
fi

if [ -z "${TAG_NAME}" ]; then
TAG_NAME=$(date +v%Y%m%d)-$(git describe --tags --always --dirty)-$(git diff | shasum -a256 | cut -c -6)
fi

# build base image
pushd ../base
./build_image.sh
popd

docker build -t ${LOCAL_IMAGE_NAME} .
if [ -z "${IMAGE_NAME}" ]; then
docker tag ${LOCAL_IMAGE_NAME} gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
docker push gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
else
docker tag ${LOCAL_IMAGE_NAME} ${IMAGE_NAME}
docker push ${IMAGE_NAME}
fi
../../build_image.sh -l ml-pipeline-dataproc-analyze "$@"
38 changes: 1 addition & 37 deletions components/dataproc/create_cluster/build_image.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,45 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.

while getopts ":hp:t:i:" opt; do
case "${opt}" in
h) echo "-p: project name"
echo "-t: tag name"
echo "-i: image name. If provided, project name and tag name are not necessary"
exit
;;
p) PROJECT_ID=${OPTARG}
;;
t) TAG_NAME=${OPTARG}
;;
i) IMAGE_NAME=${OPTARG}
;;
\? ) echo "Usage: cmd [-p] project [-t] tag [-i] image"
exit
;;
esac
done

LOCAL_IMAGE_NAME=ml-pipeline-dataproc-create-cluster

if [ -z "${PROJECT_ID}" ]; then
PROJECT_ID=$(gcloud config config-helper --format "value(configuration.properties.core.project)")
fi

if [ -z "${TAG_NAME}" ]; then
TAG_NAME=$(date +v%Y%m%d)-$(git describe --tags --always --dirty)-$(git diff | shasum -a256 | cut -c -6)
fi

# build base image
pushd ../base
./build_image.sh
popd

docker build -t ${LOCAL_IMAGE_NAME} .
if [ -z "${IMAGE_NAME}" ]; then
docker tag ${LOCAL_IMAGE_NAME} gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
docker push gcr.io/${PROJECT_ID}/${LOCAL_IMAGE_NAME}:${TAG_NAME}
else
docker tag ${LOCAL_IMAGE_NAME} ${IMAGE_NAME}
docker push ${IMAGE_NAME}
fi
../../build_image.sh -l ml-pipeline-dataproc-create-cluster "$@"
Loading

0 comments on commit fdfd25f

Please sign in to comment.