Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Add Container Engine and Image Tag parameters to backend Makefiles #10725

Merged
merged 1 commit into from
May 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions backend/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,15 @@ BUILD=build
MOD_ROOT=..
CSV_PATH=backend/third_party_licenses

# Container Build Params
CONTAINER_ENGINE ?= docker
IMG_TAG_APISERVER ?= apiserver
IMG_TAG_PERSISTENCEAGENT ?= persistence-agent
IMG_TAG_CACHESERVER ?= cache-server
IMG_TAG_SCHEDULEDWORKFLOW ?= scheduledworkflow
IMG_TAG_VIEWERCONTROLLER ?= viewercontroller
IMG_TAG_VISUALIZATION ?= visualization

# Whenever build command for any of the binaries change, we should update them both here and in backend/Dockerfiles.

.PHONY: all
Expand Down Expand Up @@ -55,19 +64,19 @@ image_all: image_apiserver image_persistence_agent image_cache image_swf image_v

.PHONY: image_apiserver
image_apiserver:
cd $(MOD_ROOT) && docker build -t apiserver -f backend/Dockerfile .
cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build -t ${IMG_TAG_APISERVER} -f backend/Dockerfile .
.PHONY: image_persistence_agent
image_persistence_agent:
cd $(MOD_ROOT) && docker build -t persistence-agent -f backend/Dockerfile.persistenceagent .
cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build -t ${IMG_TAG_PERSISTENCEAGENT} -f backend/Dockerfile.persistenceagent .
.PHONY: image_cache
image_cache:
cd $(MOD_ROOT) && docker build -t cache-server -f backend/Dockerfile.cacheserver .
cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build -t ${IMG_TAG_CACHESERVER} -f backend/Dockerfile.cacheserver .
.PHONY: image_swf
image_swf:
cd $(MOD_ROOT) && docker build -t scheduledworkflow -f backend/Dockerfile.scheduledworkflow .
cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build -t ${IMG_TAG_SCHEDULEDWORKFLOW} -f backend/Dockerfile.scheduledworkflow .
.PHONY: image_viewer
image_viewer:
cd $(MOD_ROOT) && docker build -t viewercontroller -f backend/Dockerfile.viewercontroller .
cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build -t ${IMG_TAG_VIEWERCONTROLLER} -f backend/Dockerfile.viewercontroller .
.PHONY: image_visualization
image_visualization:
cd $(MOD_ROOT) && docker build -t visualization -f backend/Dockerfile.visualization .
cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build -t ${IMG_TAG_VISUALIZATION} -f backend/Dockerfile.visualization .
12 changes: 7 additions & 5 deletions backend/api/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,12 @@ REMOTE_IMAGE=gcr.io/ml-pipeline-test/api-generator
# Keep in sync with the version used in test/release/Dockerfile.release
PREBUILT_REMOTE_IMAGE=gcr.io/ml-pipeline-test/api-generator@sha256:41fd3e60ba40430a4c3d87e03be817c5f63b2dfed23059ec9d6bca62ce0cc39c

CONTAINER_ENGINE ?= docker

# Generate clients using a pre-built api-generator image.
.PHONY: generate
generate: fetch-dependencies hack/generator.sh $(API_VERSION)/*.proto
docker run --interactive --rm \
${CONTAINER_ENGINE} run --interactive --rm \
-e API_VERSION=$(API_VERSION) \
--user $$(id -u):$$(id -g) \
--mount type=bind,source="$$(pwd)/../..",target=/go/src/github.com/kubeflow/pipelines \
Expand All @@ -44,7 +46,7 @@ v2beta1/google/rpc/status.proto:
# we should push the new image remotely to ensure everyone is using the same tools.
.PHONY: generate-from-scratch
generate-from-scratch: .image-built hack/generator.sh $(API_VERSION)/*.proto
docker run --interactive --rm \
${CONTAINER_ENGINE} run --interactive --rm \
-e API_VERSION=$(API_VERSION) \
--user $$(id -u):$$(id -g) \
--mount type=bind,source="$$(pwd)/../..",target=/go/src/github.com/kubeflow/pipelines \
Expand All @@ -57,10 +59,10 @@ image: .image-built
# Push api-generator image remotely.
.PHONY: push
push: image
docker tag $(IMAGE_TAG) $(REMOTE_IMAGE)
docker push $(REMOTE_IMAGE)
${CONTAINER_ENGINE} tag $(IMAGE_TAG) $(REMOTE_IMAGE)
${CONTAINER_ENGINE} push $(REMOTE_IMAGE)

# .image-built is a local token file to help Make determine the latest successful build.
.image-built: Dockerfile
docker build ../.. -t $(IMAGE_TAG) -f Dockerfile
${CONTAINER_ENGINE} build ../.. -t $(IMAGE_TAG) -f Dockerfile
touch .image-built
8 changes: 5 additions & 3 deletions test/imagebuilder/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

IMG = gcr.io/ml-pipeline-test/image-builder

CONTAINER_ENGINE ?= docker

# List any changed files. We only include files in the notebooks directory.
# because that is the code in the docker image.
# In particular we exclude changes to the ksonnet configs.
Expand All @@ -31,17 +33,17 @@ all: build
# To build without the cache set the environment variable
# export DOCKER_BUILD_OPTS=--no-cache
build:
docker build ${DOCKER_BUILD_OPTS} -t $(IMG):$(TAG) . \
${CONTAINER_ENGINE} build ${DOCKER_BUILD_OPTS} -t $(IMG):$(TAG) . \
--label=git-verions=$(GIT_VERSION)
docker tag $(IMG):$(TAG) $(IMG):latest
${CONTAINER_ENGINE} tag $(IMG):$(TAG) $(IMG):latest
echo Built $(IMG):$(TAG)
echo Built $(IMG):latest

# Build but don't attach the latest tag. This allows manual testing/inspection of the image
# first.
push: build
gcloud auth configure-docker
docker push $(IMG):$(TAG)
${CONTAINER_ENGINE} push $(IMG):$(TAG)
echo Pushed $(IMG):$(TAG)

push-latest: push
Expand Down
8 changes: 5 additions & 3 deletions test/release/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

REMOTE=gcr.io/ml-pipeline-test/release@sha256:ed1a4dbe536e7e161ad0d846b5681aacc0e0e7f285985cb1808c5c8987bcfeb0

CONTAINER_ENGINE ?= docker

.PHONY: release
release:
# Usage: TAG=<TAG> BRANCH=<BRANCH> make release
Expand All @@ -33,17 +35,17 @@ release-in-place:
# Build a release image locally using docker.
.PHONY: build
build:
docker build -t $(REMOTE) - < Dockerfile.release
${CONTAINER_ENGINE} build -t $(REMOTE) - < Dockerfile.release

# Push locally built release image to remote container registry,
# so that others can use the new image next time.
.PHONY: push
push: build
# Only some maintainers have access to push,
# contact @chensun @zijianjoy if you have any needs.
docker push $(REMOTE)
${CONTAINER_ENGINE} push $(REMOTE)

# Run the docker image interactively in shell as current user.
.PHONY: dev
dev:
docker run -it -u $$(id -u):$$(id -g) $(REMOTE) /bin/bash
${CONTAINER_ENGINE} run -it -u $$(id -u):$$(id -g) $(REMOTE) /bin/bash