diff --git a/boilerplate/flyte/update.sh b/boilerplate/flyte/update.sh index 506c3221b7..748ab8c9bf 100755 --- a/boilerplate/flyte/update.sh +++ b/boilerplate/flyte/update.sh @@ -11,4 +11,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" # Clone the config.yml file echo " - copying ${DIR}/config.yml to the root directory." -cp ${DIR}/config.yml ${DIR}/../../.github/config.yml +cp "${DIR}"/config.yml "${DIR}"/../../.github/config.yml diff --git a/boilerplate/update.sh b/boilerplate/update.sh index 374e4b930e..ac33ea9a70 100755 --- a/boilerplate/update.sh +++ b/boilerplate/update.sh @@ -10,7 +10,7 @@ set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" OUT="$(mktemp -d)" -trap "rm -fr $OUT" EXIT +trap 'rm -fr $OUT' EXIT git clone git@github.com:flyteorg/boilerplate.git "${OUT}" @@ -30,12 +30,12 @@ if [ ! -f "$CONFIG_FILE" ]; then fi if [ -z "$REPOSITORY" ]; then - echo '$REPOSITORY is required to run this script' + echo "$REPOSITORY is required to run this script" echo "See $README for more details." exit 1 fi -while read directory junk; do +while read -r directory junk; do # Skip comment lines (which can have leading whitespace) if [[ "$directory" == '#'* ]]; then continue @@ -63,8 +63,8 @@ while read directory junk; do echo "$directory is configured in update.cfg." echo "-----------------------------------------------------------------------------------" echo "syncing files from source." - rm -rf "${DIR}/${directory}" - mkdir -p $(dirname "${DIR}/${directory}") + rm -rf "${DIR:?}/${directory}" + mkdir -p "$(dirname "${DIR}"/"${directory}")" cp -r "$dir_path" "${DIR}/${directory}" if [ -f "${DIR}/${directory}/update.sh" ]; then echo "executing ${DIR}/${directory}/update.sh" @@ -72,4 +72,4 @@ while read directory junk; do fi echo "***********************************************************************************" echo "" -done < "$CONFIG_FILE" \ No newline at end of file +done < "$CONFIG_FILE" diff --git a/cookbook/Makefile b/cookbook/Makefile index bb91e00ec4..51bf494614 100644 --- a/cookbook/Makefile +++ b/cookbook/Makefile @@ -1,6 +1,7 @@ .SILENT: include common/Makefile +include common/parent.mk dev-requirements.txt: export CUSTOM_COMPILE_COMMAND := $(MAKE) dev-requirements.txt dev-requirements.txt: dev-requirements.in install-piptools @@ -24,55 +25,6 @@ enter_sandbox: docker_build run-examples: ## Runs all examples sh scripts/run-all-examples.sh -.PHONY: all_fast_register -all_fast_register: ## Registers new code changes using the last built image (assumes current HEAD refers to a built image). - $(MAKE) -C core fast_register - $(MAKE) -C plugins all_fast_register - $(MAKE) -C case_studies all_fast_register - -.PHONY: all_fast_serialize -all_fast_serialize: - $(MAKE) -C core fast_serialize - $(MAKE) -C plugins all_fast_serialize - $(MAKE) -C case_studies all_fast_serialize - -.PHONY: all_requirements -all_requirements: dev-requirements docs-requirements ## Makes all requirements in sub directories - $(MAKE) -C core requirements - $(MAKE) -C plugins all_requirements - $(MAKE) -C case_studies all_requirements - -.PHONY: all_register -all_register: ## Builds, pushes and registers all docker images, workflows and tasks in all sub directories. - $(MAKE) -j10 -C core register - $(MAKE) -j10 -C plugins all_register - $(MAKE) -j10 -C case_studies all_register - -.PHONY: all_serialize -all_serialize: # Builds and serializes all docker images, workflows and tasks in all sub directories. - $(MAKE) -C core serialize - $(MAKE) -C plugins all_serialize - $(MAKE) -C case_studies all_serialize - -.PHONY: all_docker_push -all_docker_push: # Builds and pushes all docker images. - $(MAKE) -j10 -C core docker_push - $(MAKE) -j10 -C plugins all_docker_push - $(MAKE) -j10 -C case_studies all_docker_push - -.PHONY: all_k3d_load_image -all_k3d_load_image: - $(MAKE) -j10 -C core k3d_load_image - $(MAKE) -j10 -C plugins all_k3d_load_image - $(MAKE) -j10 -C case_studies all_k3d_load_image - -.PHONY: all_clean -all_clean: # Cleans build directories (e.g. _pb_output/) - $(MAKE) -C core clean - $(MAKE) -C docs clean - $(MAKE) -C plugins all_clean - $(MAKE) -C case_studies all_clean - .PHONY: archive archive: # archive directories (e.g. _pb_output/) - sh scripts/archive-all-example.sh \ No newline at end of file + sh scripts/archive-all-example.sh diff --git a/cookbook/case_studies/house_price_prediction/Makefile b/cookbook/case_studies/house_price_prediction/Makefile index 92886fafe5..a510814f9d 100644 --- a/cookbook/case_studies/house_price_prediction/Makefile +++ b/cookbook/case_studies/house_price_prediction/Makefile @@ -1,2 +1,3 @@ PREFIX=house_price_prediction include ../../common/Makefile +include ../../common/leaf.mk diff --git a/cookbook/case_studies/house_price_prediction/requirements.txt b/cookbook/case_studies/house_price_prediction/requirements.txt index 763c52d848..e3a3cf879b 100644 --- a/cookbook/case_studies/house_price_prediction/requirements.txt +++ b/cookbook/case_studies/house_price_prediction/requirements.txt @@ -12,13 +12,13 @@ chardet==4.0.0 # via requests click==7.1.2 # via flytekit -croniter==1.0.8 +croniter==1.0.12 # via flytekit cycler==0.10.0 # via matplotlib -dataclasses-json==0.5.2 +dataclasses-json==0.5.3 # via flytekit -decorator==4.4.2 +decorator==5.0.7 # via retry deprecated==1.2.12 # via flytekit @@ -26,43 +26,39 @@ dirhash==0.2.1 # via flytekit docker-image-py==0.1.10 # via flytekit -flyteidl==0.18.25 +flyteidl==0.18.41 # via flytekit -flytekit==0.17.0b1 +flytekit==0.18.0 # via -r ../../common/requirements-common.in -future==0.18.2 - # via croniter -grpcio==1.36.1 +grpcio==1.37.1 # via flytekit idna==2.10 # via requests -importlib-metadata==3.7.3 +importlib-metadata==4.0.1 # via keyring joblib==1.0.1 # via # -r requirements.in # scikit-learn -keyring==23.0.0 +keyring==23.0.1 # via flytekit kiwisolver==1.3.1 # via matplotlib marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow==3.10.0 +marshmallow==3.11.1 # via # dataclasses-json # marshmallow-enum -matplotlib==3.3.4 +matplotlib==3.4.1 # via # -r ../../common/requirements-common.in # -r requirements.in mypy-extensions==0.4.3 # via typing-inspect natsort==7.1.1 - # via - # croniter - # flytekit -numpy==1.20.1 + # via flytekit +numpy==1.20.2 # via # matplotlib # pandas @@ -70,13 +66,13 @@ numpy==1.20.1 # scikit-learn # scipy # xgboost -pandas==1.2.3 +pandas==1.2.4 # via flytekit pathspec==0.8.1 # via scantree -pillow==8.1.2 +pillow==8.2.0 # via matplotlib -protobuf==3.15.6 +protobuf==3.15.8 # via # flyteidl # flytekit @@ -98,25 +94,25 @@ pytz==2018.4 # via # flytekit # pandas -regex==2021.3.17 +regex==2021.4.4 # via docker-image-py requests==2.25.1 # via # flytekit # responses -responses==0.13.1 +responses==0.13.3 # via flytekit retry==0.9.2 # via flytekit scantree==0.0.1 # via dirhash -scikit-learn==0.24.1 +scikit-learn==0.24.2 # via sklearn -scipy==1.6.1 +scipy==1.6.3 # via # scikit-learn # xgboost -six==1.15.0 +six==1.16.0 # via # cycler # flytekit @@ -137,7 +133,7 @@ tabulate==0.8.9 # via -r requirements.in threadpoolctl==2.1.0 # via scikit-learn -typing-extensions==3.7.4.3 +typing-extensions==3.10.0.0 # via typing-inspect typing-inspect==0.6.0 # via dataclasses-json @@ -154,7 +150,7 @@ wrapt==1.12.1 # via # deprecated # flytekit -xgboost==1.3.3 +xgboost==1.4.1 # via -r requirements.in zipp==3.4.1 # via importlib-metadata diff --git a/cookbook/case_studies/ml_training/Makefile b/cookbook/case_studies/ml_training/Makefile new file mode 100644 index 0000000000..80a531157c --- /dev/null +++ b/cookbook/case_studies/ml_training/Makefile @@ -0,0 +1,2 @@ +include ../common/Makefile +include ../common/parent.mk \ No newline at end of file diff --git a/cookbook/case_studies/ml_training/house_price_prediction/Makefile b/cookbook/case_studies/ml_training/house_price_prediction/Makefile index 51b39a17e3..52fa773cd8 100644 --- a/cookbook/case_studies/ml_training/house_price_prediction/Makefile +++ b/cookbook/case_studies/ml_training/house_price_prediction/Makefile @@ -1,2 +1,3 @@ PREFIX=house_price_prediction include ../../../common/Makefile +include ../../../common/leaf.mk diff --git a/cookbook/case_studies/ml_training/house_price_prediction/requirements.in b/cookbook/case_studies/ml_training/house_price_prediction/requirements.in index 1313a2c706..99af43df6f 100644 --- a/cookbook/case_studies/ml_training/house_price_prediction/requirements.in +++ b/cookbook/case_studies/ml_training/house_price_prediction/requirements.in @@ -1,4 +1,4 @@ --r ../../common/requirements-common.in +-r ../../../common/requirements-common.in xgboost joblib sklearn diff --git a/cookbook/case_studies/ml_training/pima_diabetes/Makefile b/cookbook/case_studies/ml_training/pima_diabetes/Makefile index 1722f644c0..11c64d6b09 100644 --- a/cookbook/case_studies/ml_training/pima_diabetes/Makefile +++ b/cookbook/case_studies/ml_training/pima_diabetes/Makefile @@ -1,2 +1,3 @@ PREFIX=pima_diabetes include ../../../common/Makefile +include ../../../common/leaf.mk diff --git a/cookbook/common/Makefile b/cookbook/common/Makefile index e3a7019073..75ca5a3a57 100644 --- a/cookbook/common/Makefile +++ b/cookbook/common/Makefile @@ -1,65 +1,3 @@ -.SILENT: - -# This is used by the image building script referenced below. Normally it just takes the directory name but in this -# case we want it to be called something else. -IMAGE_NAME=flytecookbook -export VERSION ?= $(shell git rev-parse HEAD) - -define PIP_COMPILE -pip-compile $(1) ${PIP_ARGS} --upgrade --verbose -endef - -# Set SANDBOX=1 to automatically fill in sandbox config -ifdef SANDBOX - -# The url for Flyte Control plane -export FLYTE_HOST ?= localhost:30081 - -# Overrides s3 url. This is solely needed for SANDBOX deployments. Shouldn't be overriden in production AWS S3. -export FLYTE_AWS_ENDPOINT ?= http://localhost:30084/ - -# Used to authenticate to s3. For a production AWS S3, it's discouraged to use keys and key ids. -export FLYTE_AWS_ACCESS_KEY_ID ?= minio - -# Used to authenticate to s3. For a production AWS S3, it's discouraged to use keys and key ids. -export FLYTE_AWS_SECRET_ACCESS_KEY ?= miniostorage - -# Used to publish artifacts for fast registration -export ADDL_DISTRIBUTION_DIR ?= s3://my-s3-bucket/fast/ - -# The base of where Blobs, Schemas and other offloaded types are, by default, serialized. -export OUTPUT_DATA_PREFIX ?= s3://my-s3-bucket/raw-data - -# Instructs flyte-cli commands to use insecure channel when communicating with Flyte's control plane. -# If you're port-forwarding your service or running the sandbox Flyte deployment, specify INSECURE=1 before your make command. -# If your Flyte Admin is behind SSL, don't specify anything. -ifndef INSECURE - export INSECURE_FLAG=-i -endif - -# The docker registry that should be used to push images. -# e.g.: -# export REGISTRY ?= ghcr.io/flyteorg -endif - -# The Flyte project that we want to register under -export PROJECT ?= flytesnacks - -# If the REGISTRY environment variable has been set, that means the image name will not just be tagged as -# flytecookbook: but rather, -# ghcr.io/flyteorg/flytecookbook: or whatever your REGISTRY is. -ifdef REGISTRY - FULL_IMAGE_NAME = ${REGISTRY}/${IMAGE_NAME} -endif -ifndef REGISTRY - FULL_IMAGE_NAME = ${IMAGE_NAME} -endif - -# If you are using a different service account on your k8s cluster, add SERVICE_ACCOUNT=my_account before your make command -ifndef SERVICE_ACCOUNT - SERVICE_ACCOUNT=default -endif - .PHONY: help help: ## show help message @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[$$()% a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -99,108 +37,3 @@ setup: install-piptools # Install requirements .PHONY: lint lint: # Run linters flake8 . - -requirements.txt: export CUSTOM_COMPILE_COMMAND := $(MAKE) requirements.txt -requirements.txt: requirements.in install-piptools - $(call PIP_COMPILE,requirements.in) - -.PHONY: requirements -requirements: requirements.txt - -.PHONY: fast_serialize -fast_serialize: clean _pb_output - echo ${CURDIR} - docker run -it --rm \ - -e REGISTRY=${REGISTRY} \ - -e MAKEFLAGS=${MAKEFLAGS} \ - -e FLYTE_HOST=${FLYTE_HOST} \ - -e INSECURE_FLAG=${INSECURE_FLAG} \ - -e PROJECT=${PROJECT} \ - -e FLYTE_AWS_ENDPOINT=${FLYTE_AWS_ENDPOINT} \ - -e FLYTE_AWS_ACCESS_KEY_ID=${FLYTE_AWS_ACCESS_KEY_ID} \ - -e FLYTE_AWS_SECRET_ACCESS_KEY=${FLYTE_AWS_SECRET_ACCESS_KEY} \ - -e OUTPUT_DATA_PREFIX=${OUTPUT_DATA_PREFIX} \ - -e ADDL_DISTRIBUTION_DIR=${ADDL_DISTRIBUTION_DIR} \ - -e SERVICE_ACCOUNT=$(SERVICE_ACCOUNT) \ - -e VERSION=${VERSION} \ - -v ${CURDIR}/_pb_output:/tmp/output \ - -v ${CURDIR}:/root/$(shell basename $(CURDIR)) \ - ${TAGGED_IMAGE} make fast_serialize - -.PHONY: fast_register -fast_register: clean _pb_output ## Packages code and registers without building docker images. - @echo "Tagged Image: " - @echo ${TAGGED_IMAGE} - @echo ${CURDIR} - docker run -it --rm \ - --network host \ - -e REGISTRY=${REGISTRY} \ - -e MAKEFLAGS=${MAKEFLAGS} \ - -e FLYTE_HOST=${FLYTE_HOST} \ - -e INSECURE_FLAG=${INSECURE_FLAG} \ - -e PROJECT=${PROJECT} \ - -e FLYTE_AWS_ENDPOINT=${FLYTE_AWS_ENDPOINT} \ - -e FLYTE_AWS_ACCESS_KEY_ID=${FLYTE_AWS_ACCESS_KEY_ID} \ - -e FLYTE_AWS_SECRET_ACCESS_KEY=${FLYTE_AWS_SECRET_ACCESS_KEY} \ - -e OUTPUT_DATA_PREFIX=${OUTPUT_DATA_PREFIX} \ - -e ADDL_DISTRIBUTION_DIR=${ADDL_DISTRIBUTION_DIR} \ - -e SERVICE_ACCOUNT=$(SERVICE_ACCOUNT) \ - -e VERSION=${VERSION} \ - -v ${CURDIR}/_pb_output:/tmp/output \ - -v ${CURDIR}:/root/$(shell basename $(CURDIR)) \ - ${TAGGED_IMAGE} make fast_register - -.PHONY: docker_build -docker_build: - echo "Tagged Image: " - echo ${TAGGED_IMAGE} - docker build ../ --build-arg tag="${TAGGED_IMAGE}" -t "${TAGGED_IMAGE}" -f Dockerfile - -.PHONY: serialize -serialize: clean _pb_output docker_build - @echo ${VERSION} - @echo ${CURDIR} - docker run -i --rm \ - -e REGISTRY=${REGISTRY} \ - -e MAKEFLAGS=${MAKEFLAGS} \ - -e FLYTE_HOST=${FLYTE_HOST} \ - -e INSECURE_FLAG=${INSECURE_FLAG} \ - -e PROJECT=${PROJECT} \ - -e FLYTE_AWS_ENDPOINT=${FLYTE_AWS_ENDPOINT} \ - -e FLYTE_AWS_ACCESS_KEY_ID=${FLYTE_AWS_ACCESS_KEY_ID} \ - -e FLYTE_AWS_SECRET_ACCESS_KEY=${FLYTE_AWS_SECRET_ACCESS_KEY} \ - -e OUTPUT_DATA_PREFIX=${OUTPUT_DATA_PREFIX} \ - -e ADDL_DISTRIBUTION_DIR=${ADDL_DISTRIBUTION_DIR} \ - -e SERVICE_ACCOUNT=$(SERVICE_ACCOUNT) \ - -e VERSION=${VERSION} \ - -v ${CURDIR}/_pb_output:/tmp/output \ - ${TAGGED_IMAGE} make serialize - - -.PHONY: register -register: clean _pb_output docker_push - @echo ${VERSION} - @echo ${CURDIR} - docker run -i --rm \ - --network host \ - -e REGISTRY=${REGISTRY} \ - -e MAKEFLAGS=${MAKEFLAGS} \ - -e FLYTE_HOST=${FLYTE_HOST} \ - -e INSECURE_FLAG=${INSECURE_FLAG} \ - -e PROJECT=${PROJECT} \ - -e FLYTE_AWS_ENDPOINT=${FLYTE_AWS_ENDPOINT} \ - -e FLYTE_AWS_ACCESS_KEY_ID=${FLYTE_AWS_ACCESS_KEY_ID} \ - -e FLYTE_AWS_SECRET_ACCESS_KEY=${FLYTE_AWS_SECRET_ACCESS_KEY} \ - -e OUTPUT_DATA_PREFIX=${OUTPUT_DATA_PREFIX} \ - -e ADDL_DISTRIBUTION_DIR=${ADDL_DISTRIBUTION_DIR} \ - -e SERVICE_ACCOUNT=$(SERVICE_ACCOUNT) \ - -e VERSION=${VERSION} \ - -v ${CURDIR}/_pb_output:/tmp/output \ - ${TAGGED_IMAGE} make register - -_pb_output: - mkdir -p _pb_output - -.PHONY: clean -clean: - rm -rf _pb_output/* diff --git a/cookbook/common/leaf.mk b/cookbook/common/leaf.mk new file mode 100644 index 0000000000..a58eca34be --- /dev/null +++ b/cookbook/common/leaf.mk @@ -0,0 +1,166 @@ +.SILENT: + +# This is used by the image building script referenced below. Normally it just takes the directory name but in this +# case we want it to be called something else. +IMAGE_NAME=flytecookbook +export VERSION ?= $(shell git rev-parse HEAD) + +define PIP_COMPILE +pip-compile $(1) ${PIP_ARGS} --upgrade --verbose +endef + +# Set SANDBOX=1 to automatically fill in sandbox config +ifdef SANDBOX + +# The url for Flyte Control plane +export FLYTE_HOST ?= localhost:30081 + +# Overrides s3 url. This is solely needed for SANDBOX deployments. Shouldn't be overriden in production AWS S3. +export FLYTE_AWS_ENDPOINT ?= http://localhost:30084/ + +# Used to authenticate to s3. For a production AWS S3, it's discouraged to use keys and key ids. +export FLYTE_AWS_ACCESS_KEY_ID ?= minio + +# Used to authenticate to s3. For a production AWS S3, it's discouraged to use keys and key ids. +export FLYTE_AWS_SECRET_ACCESS_KEY ?= miniostorage + +# Used to publish artifacts for fast registration +export ADDL_DISTRIBUTION_DIR ?= s3://my-s3-bucket/fast/ + +# The base of where Blobs, Schemas and other offloaded types are, by default, serialized. +export OUTPUT_DATA_PREFIX ?= s3://my-s3-bucket/raw-data + +# Instructs flyte-cli commands to use insecure channel when communicating with Flyte's control plane. +# If you're port-forwarding your service or running the sandbox Flyte deployment, specify INSECURE=1 before your make command. +# If your Flyte Admin is behind SSL, don't specify anything. +ifndef INSECURE + export INSECURE_FLAG=-i +endif + +# The docker registry that should be used to push images. +# e.g.: +# export REGISTRY ?= ghcr.io/flyteorg +endif + +# The Flyte project that we want to register under +export PROJECT ?= flytesnacks + +# If the REGISTRY environment variable has been set, that means the image name will not just be tagged as +# flytecookbook: but rather, +# ghcr.io/flyteorg/flytecookbook: or whatever your REGISTRY is. +ifdef REGISTRY + FULL_IMAGE_NAME = ${REGISTRY}/${IMAGE_NAME} +endif +ifndef REGISTRY + FULL_IMAGE_NAME = ${IMAGE_NAME} +endif + +# If you are using a different service account on your k8s cluster, add SERVICE_ACCOUNT=my_account before your make command +ifndef SERVICE_ACCOUNT + SERVICE_ACCOUNT=default +endif + +requirements.txt: export CUSTOM_COMPILE_COMMAND := $(MAKE) requirements.txt +requirements.txt: requirements.in install-piptools + $(call PIP_COMPILE,requirements.in) + +.PHONY: requirements +requirements: requirements.txt + +.PHONY: fast_serialize +fast_serialize: clean _pb_output + echo ${CURDIR} + docker run -it --rm \ + -e REGISTRY=${REGISTRY} \ + -e MAKEFLAGS=${MAKEFLAGS} \ + -e FLYTE_HOST=${FLYTE_HOST} \ + -e INSECURE_FLAG=${INSECURE_FLAG} \ + -e PROJECT=${PROJECT} \ + -e FLYTE_AWS_ENDPOINT=${FLYTE_AWS_ENDPOINT} \ + -e FLYTE_AWS_ACCESS_KEY_ID=${FLYTE_AWS_ACCESS_KEY_ID} \ + -e FLYTE_AWS_SECRET_ACCESS_KEY=${FLYTE_AWS_SECRET_ACCESS_KEY} \ + -e OUTPUT_DATA_PREFIX=${OUTPUT_DATA_PREFIX} \ + -e ADDL_DISTRIBUTION_DIR=${ADDL_DISTRIBUTION_DIR} \ + -e SERVICE_ACCOUNT=$(SERVICE_ACCOUNT) \ + -e VERSION=${VERSION} \ + -v ${CURDIR}/_pb_output:/tmp/output \ + -v ${CURDIR}:/root/$(shell basename $(CURDIR)) \ + ${TAGGED_IMAGE} make fast_serialize + +.PHONY: fast_register +fast_register: clean _pb_output ## Packages code and registers without building docker images. + @echo "Tagged Image: " + @echo ${TAGGED_IMAGE} + @echo ${CURDIR} + docker run -it --rm \ + --network host \ + -e REGISTRY=${REGISTRY} \ + -e MAKEFLAGS=${MAKEFLAGS} \ + -e FLYTE_HOST=${FLYTE_HOST} \ + -e INSECURE_FLAG=${INSECURE_FLAG} \ + -e PROJECT=${PROJECT} \ + -e FLYTE_AWS_ENDPOINT=${FLYTE_AWS_ENDPOINT} \ + -e FLYTE_AWS_ACCESS_KEY_ID=${FLYTE_AWS_ACCESS_KEY_ID} \ + -e FLYTE_AWS_SECRET_ACCESS_KEY=${FLYTE_AWS_SECRET_ACCESS_KEY} \ + -e OUTPUT_DATA_PREFIX=${OUTPUT_DATA_PREFIX} \ + -e ADDL_DISTRIBUTION_DIR=${ADDL_DISTRIBUTION_DIR} \ + -e SERVICE_ACCOUNT=$(SERVICE_ACCOUNT) \ + -e VERSION=${VERSION} \ + -v ${CURDIR}/_pb_output:/tmp/output \ + -v ${CURDIR}:/root/$(shell basename $(CURDIR)) \ + ${TAGGED_IMAGE} make fast_register + +.PHONY: docker_build +docker_build: + echo "Tagged Image: " + echo ${TAGGED_IMAGE} + docker build ../ --build-arg tag="${TAGGED_IMAGE}" -t "${TAGGED_IMAGE}" -f Dockerfile + +.PHONY: serialize +serialize: clean _pb_output docker_build + @echo ${VERSION} + @echo ${CURDIR} + docker run -i --rm \ + -e REGISTRY=${REGISTRY} \ + -e MAKEFLAGS=${MAKEFLAGS} \ + -e FLYTE_HOST=${FLYTE_HOST} \ + -e INSECURE_FLAG=${INSECURE_FLAG} \ + -e PROJECT=${PROJECT} \ + -e FLYTE_AWS_ENDPOINT=${FLYTE_AWS_ENDPOINT} \ + -e FLYTE_AWS_ACCESS_KEY_ID=${FLYTE_AWS_ACCESS_KEY_ID} \ + -e FLYTE_AWS_SECRET_ACCESS_KEY=${FLYTE_AWS_SECRET_ACCESS_KEY} \ + -e OUTPUT_DATA_PREFIX=${OUTPUT_DATA_PREFIX} \ + -e ADDL_DISTRIBUTION_DIR=${ADDL_DISTRIBUTION_DIR} \ + -e SERVICE_ACCOUNT=$(SERVICE_ACCOUNT) \ + -e VERSION=${VERSION} \ + -v ${CURDIR}/_pb_output:/tmp/output \ + ${TAGGED_IMAGE} make serialize + + +.PHONY: register +register: clean _pb_output docker_push + @echo ${VERSION} + @echo ${CURDIR} + docker run -i --rm \ + --network host \ + -e REGISTRY=${REGISTRY} \ + -e MAKEFLAGS=${MAKEFLAGS} \ + -e FLYTE_HOST=${FLYTE_HOST} \ + -e INSECURE_FLAG=${INSECURE_FLAG} \ + -e PROJECT=${PROJECT} \ + -e FLYTE_AWS_ENDPOINT=${FLYTE_AWS_ENDPOINT} \ + -e FLYTE_AWS_ACCESS_KEY_ID=${FLYTE_AWS_ACCESS_KEY_ID} \ + -e FLYTE_AWS_SECRET_ACCESS_KEY=${FLYTE_AWS_SECRET_ACCESS_KEY} \ + -e OUTPUT_DATA_PREFIX=${OUTPUT_DATA_PREFIX} \ + -e ADDL_DISTRIBUTION_DIR=${ADDL_DISTRIBUTION_DIR} \ + -e SERVICE_ACCOUNT=$(SERVICE_ACCOUNT) \ + -e VERSION=${VERSION} \ + -v ${CURDIR}/_pb_output:/tmp/output \ + ${TAGGED_IMAGE} make register + +_pb_output: + mkdir -p _pb_output + +.PHONY: clean +clean: + rm -rf _pb_output/* diff --git a/cookbook/common/parent.mk b/cookbook/common/parent.mk index 0bc7da9aa0..4b849ae283 100644 --- a/cookbook/common/parent.mk +++ b/cookbook/common/parent.mk @@ -9,65 +9,68 @@ PWD=$(CURDIR) help: ## show help message @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n $(MAKE) \033[36m\033[0m\n"} /^[$$()% a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) -.PHONY: all_fast_serialize -all_fast_serialize: - for dir in $(SUBDIRS) ; do \ +.PHONY: fast_serialize +fast_serialize: + @for dir in $(SUBDIRS) ; do \ + echo "processing ${PWD}/$$dir"; \ trimmed=$${dir%/}; \ - test -f $$dir/Dockerfile && \ + test -f $$dir/Makefile && \ PREFIX=$$trimmed $(MAKE) fast_serialize; \ done -.PHONY: all_fast_register -all_fast_register: ## Registers new code changes using the last built image (assumes current HEAD refers to a built image). - for dir in $(SUBDIRS) ; do \ +.PHONY: fast_register +fast_register: ## Registers new code changes using the last built image (assumes current HEAD refers to a built image). + @for dir in $(SUBDIRS) ; do \ + echo "processing ${PWD}/$$dir"; \ trimmed=$${dir%/}; \ - test -f $$dir/Dockerfile && \ + test -f $$dir/Makefile && \ PREFIX=$$trimmed $(MAKE) fast_register; \ done -.PHONY: all_register -all_register: ## Builds, pushes and registers all docker images, workflows and tasks in all sub directories. - for dir in $(SUBDIRS) ; do \ - make -C $$dir register; \ +.PHONY: register +register: ## Builds, pushes and registers all docker images, workflows and tasks in all sub directories. + @for dir in $(SUBDIRS) ; do \ + echo "processing ${PWD}/$$dir"; \ + test -f $$dir/Makefile && \ + $(MAKE) -C $$dir register; \ done -.PHONY: all_serialize -all_serialize: ## Builds and serializes all docker images, workflows and tasks in all sub directories. - echo "processing ${PWD}" - for dir in $(SUBDIRS) ; do \ - echo "processing $$dir"; \ - make -C $$dir serialize; \ +.PHONY: serialize +serialize: ## Builds and serializes all docker images, workflows and tasks in all sub directories. + @for dir in $(SUBDIRS) ; do \ + echo "processing ${PWD}/$$dir"; \ + test -f $$dir/Makefile && \ + $(MAKE) -C $$dir serialize; \ done -.PHONY: all_docker_push -all_docker_push: ## Builds and pushes all docker images. - echo "processing ${PWD}" - for dir in $(SUBDIRS) ; do \ - echo "processing $$dir"; \ - make -C $$dir docker_push; \ +.PHONY: docker_push +docker_push: ## Builds and pushes all docker images. + @for dir in $(SUBDIRS) ; do \ + echo "processing ${PWD}/$$dir"; \ + test -f $$dir/Makefile && \ + $(MAKE) -C $$dir docker_push; \ done -.PHONY: all_requirements -all_requirements: ## Makes all requirement files in sub directories. - echo "processing ${PWD}" - for dir in $(SUBDIRS) ; do \ - echo "processing $$dir"; \ - make -C $$dir requirements; \ +.PHONY: requirements +requirements: ## Makes all requirement files in sub directories. + @for dir in $(SUBDIRS) ; do \ + echo "processing ${PWD}/$$dir"; \ + test -f $$dir/Makefile && \ + $(MAKE) -C $$dir requirements; \ done -.PHONY: all_k3d_load_image -all_k3d_load_image: - echo "processing ${PWD}" - for dir in $(SUBDIRS) ; do \ - echo "processing $$dir"; \ - test -f $$dir/Dockerfile && \ - test -f $$dir/Makefile && $(MAKE) -C $$dir k3d_load_image; \ +.PHONY: k3d_load_image +k3d_load_image: + @for dir in $(SUBDIRS) ; do \ + echo "processing ${PWD}/$$dir"; \ + test -f $$dir/Makefile && \ + $(MAKE) -C $$dir k3d_load_image; \ done -.PHONY: all_clean -all_clean: ## Deletes build directories (e.g. _pb_output/) - echo "processing ${PWD}" - for dir in $(SUBDIRS) ; do \ - echo "processing $$dir"; \ - test -f $$dir/Makefile && $(MAKE) -C $$dir clean; \ +.PHONY: clean +clean: ## Deletes build directories (e.g. _pb_output/) + @for dir in $(SUBDIRS) ; do \ + echo "processing ${PWD}/$$dir"; \ + test -f $$dir/Makefile && \ + $(MAKE) -C $$dir clean; \ done diff --git a/cookbook/common/requirements-common.in b/cookbook/common/requirements-common.in index dc31ec0e5e..7c543bcb47 100644 --- a/cookbook/common/requirements-common.in +++ b/cookbook/common/requirements-common.in @@ -1,3 +1,3 @@ -flytekit>=0.17.0b0 +flytekit>=0.18.0 wheel matplotlib diff --git a/cookbook/core/Makefile b/cookbook/core/Makefile index a408d467b0..dba6e81429 100644 --- a/cookbook/core/Makefile +++ b/cookbook/core/Makefile @@ -1,2 +1,3 @@ PREFIX=core include ../common/Makefile +include ../common/leaf.mk diff --git a/cookbook/core/requirements.txt b/cookbook/core/requirements.txt index 400c5001b7..8bbe0b6275 100644 --- a/cookbook/core/requirements.txt +++ b/cookbook/core/requirements.txt @@ -12,13 +12,13 @@ chardet==4.0.0 # via requests click==7.1.2 # via flytekit -croniter==1.0.8 +croniter==1.0.12 # via flytekit cycler==0.10.0 # via matplotlib -dataclasses-json==0.5.2 +dataclasses-json==0.5.3 # via flytekit -decorator==4.4.2 +decorator==5.0.7 # via retry deprecated==1.2.12 # via flytekit @@ -26,37 +26,33 @@ dirhash==0.2.1 # via flytekit docker-image-py==0.1.10 # via flytekit -flyteidl==0.18.25 +flyteidl==0.18.41 # via flytekit -flytekit==0.17.0b0 +flytekit==0.18.0 # via -r ../common/requirements-common.in -future==0.18.2 - # via croniter -grpcio==1.36.1 +grpcio==1.37.1 # via flytekit idna==2.10 # via requests -importlib-metadata==3.7.3 +importlib-metadata==4.0.1 # via keyring -keyring==23.0.0 +keyring==23.0.1 # via flytekit kiwisolver==1.3.1 # via matplotlib marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow==3.10.0 +marshmallow==3.11.1 # via # dataclasses-json # marshmallow-enum -matplotlib==3.3.4 +matplotlib==3.4.1 # via -r ../common/requirements-common.in mypy-extensions==0.4.3 # via typing-inspect natsort==7.1.1 - # via - # croniter - # flytekit -numpy==1.20.1 + # via flytekit +numpy==1.20.2 # via # matplotlib # opencv-python @@ -64,13 +60,13 @@ numpy==1.20.1 # pyarrow opencv-python==4.5.1.48 # via -r requirements.in -pandas==1.2.3 +pandas==1.2.4 # via flytekit pathspec==0.8.1 # via scantree -pillow==8.1.2 +pillow==8.2.0 # via matplotlib -protobuf==3.15.6 +protobuf==3.15.8 # via # flyteidl # flytekit @@ -92,19 +88,19 @@ pytz==2018.4 # via # flytekit # pandas -regex==2021.3.17 +regex==2021.4.4 # via docker-image-py requests==2.25.1 # via # flytekit # responses -responses==0.13.1 +responses==0.13.3 # via flytekit retry==0.9.2 # via flytekit scantree==0.0.1 # via dirhash -six==1.15.0 +six==1.16.0 # via # cycler # flytekit @@ -119,7 +115,7 @@ statsd==3.3.0 # via flytekit stringcase==1.2.0 # via dataclasses-json -typing-extensions==3.7.4.3 +typing-extensions==3.10.0.0 # via typing-inspect typing-inspect==0.6.0 # via dataclasses-json diff --git a/cookbook/dev-requirements.in b/cookbook/dev-requirements.in index 0783ed64be..421a116c1e 100644 --- a/cookbook/dev-requirements.in +++ b/cookbook/dev-requirements.in @@ -1,8 +1,8 @@ -r ./integrations/kubernetes/pod/requirements.in -r ./integrations/kubernetes/k8s_spark/requirements.in --r ./integrations/aws/sagemaker/sagemaker_training/requirements.in --r ./integrations/kubernetes/kubeflow/kfpytorch/requirements.in --r ./integrations/aws/sagemaker/sagemaker_pytorch/requirements.in +-r ./integrations/aws/sagemaker_training/requirements.in +-r ./integrations/kubernetes/kfpytorch/requirements.in +-r ./integrations/aws/sagemaker_pytorch/requirements.in -r ./integrations/external_services/hive/requirements.in -r ./case_studies/ml_training/pima_diabetes/requirements.in -r ./docs-requirements.in diff --git a/cookbook/dev-requirements.txt b/cookbook/dev-requirements.txt index a32640f69f..3ae0d5ea7d 100644 --- a/cookbook/dev-requirements.txt +++ b/cookbook/dev-requirements.txt @@ -38,9 +38,9 @@ black==19.10b0 # via # -r dev-requirements.in # flake8-black -boto3==1.17.60 +boto3==1.17.67 # via sagemaker-training -botocore==1.20.60 +botocore==1.20.67 # via # boto3 # s3transfer @@ -97,39 +97,39 @@ flake8==3.9.1 # flake8-isort flatbuffers==1.12 # via tensorflow -flyteidl==0.18.39 +flyteidl==0.18.41 # via flytekit -flytekit==0.17.0 +flytekit==0.18.0 # via # -r ./case_studies/ml_training/pima_diabetes/../../../common/requirements-common.in - # -r ./integrations/aws/sagemaker/sagemaker_pytorch/../../../../common/requirements-common.in - # -r ./integrations/aws/sagemaker/sagemaker_training/../../../../common/requirements-common.in + # -r ./integrations/aws/sagemaker_pytorch/../../../common/requirements-common.in + # -r ./integrations/aws/sagemaker_training/../../../common/requirements-common.in # -r ./integrations/external_services/hive/../../../common/requirements-common.in # -r ./integrations/kubernetes/k8s_spark/../../../common/requirements-common.in - # -r ./integrations/kubernetes/kubeflow/kfpytorch/../../../../common/requirements-common.in + # -r ./integrations/kubernetes/kfpytorch/../../../common/requirements-common.in # -r ./integrations/kubernetes/pod/../../../common/requirements-common.in # flytekitplugins-awssagemaker # flytekitplugins-hive # flytekitplugins-kfpytorch # flytekitplugins-pod # flytekitplugins-spark -flytekitplugins-awssagemaker==0.17.0 +flytekitplugins-awssagemaker==0.18.0 # via - # -r ./integrations/aws/sagemaker/sagemaker_pytorch/requirements.in - # -r ./integrations/aws/sagemaker/sagemaker_training/requirements.in -flytekitplugins-hive==0.17.0 + # -r ./integrations/aws/sagemaker_pytorch/requirements.in + # -r ./integrations/aws/sagemaker_training/requirements.in +flytekitplugins-hive==0.18.0 # via -r ./integrations/external_services/hive/requirements.in -flytekitplugins-kfpytorch==0.17.0 - # via -r ./integrations/kubernetes/kubeflow/kfpytorch/requirements.in -flytekitplugins-pod==0.17.0 +flytekitplugins-kfpytorch==0.18.0 + # via -r ./integrations/kubernetes/kfpytorch/requirements.in +flytekitplugins-pod==0.18.0 # via -r ./integrations/kubernetes/pod/requirements.in -flytekitplugins-spark==0.17.0 +flytekitplugins-spark==0.18.0 # via -r ./integrations/kubernetes/k8s_spark/requirements.in git+git://github.com/flyteorg/furo@main # via -r ./docs-requirements.in future==0.18.2 # via tensorflow-datasets -gast==0.4.0 +gast==0.3.3 # via tensorflow gevent==21.1.2 # via sagemaker-training @@ -146,12 +146,12 @@ googleapis-common-protos==1.53.0 # via tensorflow-metadata greenlet==1.0.0 # via gevent -grpcio==1.34.1 +grpcio==1.32.0 # via # flytekit # tensorboard # tensorflow -h5py==3.1.0 +h5py==2.10.0 # via tensorflow idna==2.10 # via requests @@ -159,6 +159,8 @@ imagesize==1.2.0 # via sphinx importlib-metadata==4.0.1 # via keyring +importlib-resources==5.1.2 + # via tensorflow-datasets iniconfig==1.1.1 # via pytest inotify_simple==1.2.1 @@ -179,8 +181,6 @@ joblib==1.0.1 # via # -r ./case_studies/ml_training/pima_diabetes/requirements.in # scikit-learn -keras-nightly==2.5.0.dev2021032900 - # via tensorflow keras-preprocessing==1.1.2 # via tensorflow keyring==23.0.1 @@ -205,11 +205,11 @@ matplotlib==3.4.1 # via # -r ./case_studies/ml_training/pima_diabetes/../../../common/requirements-common.in # -r ./case_studies/ml_training/pima_diabetes/requirements.in - # -r ./integrations/aws/sagemaker/sagemaker_pytorch/../../../../common/requirements-common.in - # -r ./integrations/aws/sagemaker/sagemaker_training/../../../../common/requirements-common.in + # -r ./integrations/aws/sagemaker_pytorch/../../../common/requirements-common.in + # -r ./integrations/aws/sagemaker_training/../../../common/requirements-common.in # -r ./integrations/external_services/hive/../../../common/requirements-common.in # -r ./integrations/kubernetes/k8s_spark/../../../common/requirements-common.in - # -r ./integrations/kubernetes/kubeflow/kfpytorch/../../../../common/requirements-common.in + # -r ./integrations/kubernetes/kfpytorch/../../../common/requirements-common.in # -r ./integrations/kubernetes/pod/../../../common/requirements-common.in mccabe==0.6.1 # via flake8 @@ -295,7 +295,7 @@ pycparser==2.20 # via cffi pyflakes==2.3.1 # via flake8 -pygments==2.8.1 +pygments==2.9.0 # via # sphinx # sphinx-prompt @@ -309,7 +309,7 @@ pyspark==3.1.1 # via # -r ./integrations/kubernetes/k8s_spark/requirements.in # flytekitplugins-spark -pytest==6.2.3 +pytest==6.2.4 # via -r dev-requirements.in python-dateutil==2.8.1 # via @@ -381,6 +381,7 @@ six==1.15.0 # google-auth # google-pasta # grpcio + # h5py # keras-preprocessing # kubernetes # promise @@ -443,7 +444,7 @@ stringcase==1.2.0 # via dataclasses-json tabulate==0.8.9 # via -r ./case_studies/ml_training/pima_diabetes/requirements.in -tensorboard-data-server==0.6.0 +tensorboard-data-server==0.6.1 # via tensorboard tensorboard-plugin-wit==1.8.0 # via tensorboard @@ -451,16 +452,16 @@ tensorboard==2.5.0 # via tensorflow tensorboardx==2.2 # via - # -r ./integrations/aws/sagemaker/sagemaker_pytorch/requirements.in - # -r ./integrations/kubernetes/kubeflow/kfpytorch/requirements.in + # -r ./integrations/aws/sagemaker_pytorch/requirements.in + # -r ./integrations/kubernetes/kfpytorch/requirements.in tensorflow-datasets==4.2.0 - # via -r ./integrations/aws/sagemaker/sagemaker_training/requirements.in -tensorflow-estimator==2.5.0rc0 + # via -r ./integrations/aws/sagemaker_training/requirements.in +tensorflow-estimator==2.4.0 # via tensorflow tensorflow-metadata==0.30.0 # via tensorflow-datasets -tensorflow==2.5.0rc2 - # via -r ./integrations/aws/sagemaker/sagemaker_training/requirements.in +tensorflow==2.4.1 + # via -r ./integrations/aws/sagemaker_training/requirements.in termcolor==1.1.0 # via # tensorflow @@ -475,13 +476,13 @@ toml==0.10.2 # pytest torch==1.8.1 # via - # -r ./integrations/aws/sagemaker/sagemaker_pytorch/requirements.in - # -r ./integrations/kubernetes/kubeflow/kfpytorch/requirements.in + # -r ./integrations/aws/sagemaker_pytorch/requirements.in + # -r ./integrations/kubernetes/kfpytorch/requirements.in # torchvision torchvision==0.9.1 # via - # -r ./integrations/aws/sagemaker/sagemaker_pytorch/requirements.in - # -r ./integrations/kubernetes/kubeflow/kfpytorch/requirements.in + # -r ./integrations/aws/sagemaker_pytorch/requirements.in + # -r ./integrations/kubernetes/kfpytorch/requirements.in tqdm==4.60.0 # via tensorflow-datasets typed-ast==1.4.3 @@ -502,7 +503,7 @@ urllib3==1.25.11 # kubernetes # requests # responses -websocket-client==0.58.0 +websocket-client==0.59.0 # via kubernetes werkzeug==1.0.1 # via @@ -511,11 +512,11 @@ werkzeug==1.0.1 wheel==0.36.2 # via # -r ./case_studies/ml_training/pima_diabetes/../../../common/requirements-common.in - # -r ./integrations/aws/sagemaker/sagemaker_pytorch/../../../../common/requirements-common.in - # -r ./integrations/aws/sagemaker/sagemaker_training/../../../../common/requirements-common.in + # -r ./integrations/aws/sagemaker_pytorch/../../../common/requirements-common.in + # -r ./integrations/aws/sagemaker_training/../../../common/requirements-common.in # -r ./integrations/external_services/hive/../../../common/requirements-common.in # -r ./integrations/kubernetes/k8s_spark/../../../common/requirements-common.in - # -r ./integrations/kubernetes/kubeflow/kfpytorch/../../../../common/requirements-common.in + # -r ./integrations/kubernetes/kfpytorch/../../../common/requirements-common.in # -r ./integrations/kubernetes/pod/../../../common/requirements-common.in # astunparse # flytekit diff --git a/cookbook/docs-requirements.txt b/cookbook/docs-requirements.txt index 6ba8f2c45c..c9a90de33d 100644 --- a/cookbook/docs-requirements.txt +++ b/cookbook/docs-requirements.txt @@ -43,7 +43,7 @@ packaging==20.9 # via sphinx pillow==8.2.0 # via -r docs-requirements.in -pygments==2.8.1 +pygments==2.9.0 # via # sphinx # sphinx-prompt @@ -57,7 +57,7 @@ readthedocs-sphinx-search==0.1.0 # via -r docs-requirements.in requests==2.25.1 # via sphinx -six==1.15.0 +six==1.16.0 # via # astroid # sphinx-code-include diff --git a/cookbook/integrations/aws/Makefile b/cookbook/integrations/aws/Makefile new file mode 100644 index 0000000000..488b3f6b76 --- /dev/null +++ b/cookbook/integrations/aws/Makefile @@ -0,0 +1,2 @@ +include ../../common/Makefile +include ../../common/parent.mk diff --git a/cookbook/integrations/aws/sagemaker_pytorch/Makefile b/cookbook/integrations/aws/sagemaker_pytorch/Makefile index 95ff33fa96..913f29885a 100644 --- a/cookbook/integrations/aws/sagemaker_pytorch/Makefile +++ b/cookbook/integrations/aws/sagemaker_pytorch/Makefile @@ -1,2 +1,3 @@ PREFIX=sagemaker_pytorch include ../../../common/Makefile +include ../../../common/leaf.mk diff --git a/cookbook/integrations/aws/sagemaker_training/Makefile b/cookbook/integrations/aws/sagemaker_training/Makefile index d70fc9cef1..1247319d42 100644 --- a/cookbook/integrations/aws/sagemaker_training/Makefile +++ b/cookbook/integrations/aws/sagemaker_training/Makefile @@ -1,2 +1,3 @@ PREFIX=sagemaker_training include ../../../common/Makefile +include ../../../common/leaf.mk diff --git a/cookbook/integrations/aws/sagemaker_training/requirements.in b/cookbook/integrations/aws/sagemaker_training/requirements.in index 2422c6c33f..6311ce521a 100644 --- a/cookbook/integrations/aws/sagemaker_training/requirements.in +++ b/cookbook/integrations/aws/sagemaker_training/requirements.in @@ -1,4 +1,4 @@ --r ../../../../common/requirements-common.in +-r ../../../common/requirements-common.in flytekitplugins-awssagemaker>=0.16.0 tensorflow tensorflow-datasets diff --git a/cookbook/integrations/aws/sagemaker_training/requirements.txt b/cookbook/integrations/aws/sagemaker_training/requirements.txt index b00ecb7358..8d21969e2a 100644 --- a/cookbook/integrations/aws/sagemaker_training/requirements.txt +++ b/cookbook/integrations/aws/sagemaker_training/requirements.txt @@ -18,13 +18,13 @@ attrs==20.3.0 # tensorflow-datasets bcrypt==3.2.0 # via paramiko -boto3==1.17.35 +boto3==1.17.67 # via sagemaker-training -botocore==1.20.35 +botocore==1.20.67 # via # boto3 # s3transfer -cachetools==4.2.1 +cachetools==4.2.2 # via google-auth certifi==2020.12.5 # via requests @@ -37,15 +37,15 @@ chardet==4.0.0 # via requests click==7.1.2 # via flytekit -croniter==1.0.8 +croniter==1.0.12 # via flytekit -cryptography==3.4.6 +cryptography==3.4.7 # via paramiko cycler==0.10.0 # via matplotlib -dataclasses-json==0.5.2 +dataclasses-json==0.5.3 # via flytekit -decorator==4.4.2 +decorator==5.0.7 # via retry deprecated==1.2.12 # via flytekit @@ -57,25 +57,23 @@ docker-image-py==0.1.10 # via flytekit flatbuffers==1.12 # via tensorflow -flyteidl==0.18.25 +flyteidl==0.18.41 # via flytekit -flytekit==0.17.0b0 +flytekit==0.18.0 # via - # -r ../../common/requirements-common.in + # -r ../../../common/requirements-common.in # flytekitplugins-awssagemaker -flytekitplugins-awssagemaker==0.16.0 +flytekitplugins-awssagemaker==0.18.0 # via -r requirements.in future==0.18.2 - # via - # croniter - # tensorflow-datasets + # via tensorflow-datasets gast==0.3.3 # via tensorflow gevent==21.1.2 # via sagemaker-training -google-auth-oauthlib==0.4.3 +google-auth-oauthlib==0.4.4 # via tensorboard -google-auth==1.28.0 +google-auth==1.30.0 # via # google-auth-oauthlib # tensorboard @@ -94,7 +92,7 @@ h5py==2.10.0 # via tensorflow idna==2.10 # via requests -importlib-metadata==3.7.3 +importlib-metadata==4.0.1 # via keyring importlib-resources==5.1.2 # via tensorflow-datasets @@ -106,7 +104,7 @@ jmespath==0.10.0 # botocore keras-preprocessing==1.1.2 # via tensorflow -keyring==23.0.0 +keyring==23.0.1 # via flytekit kiwisolver==1.3.1 # via matplotlib @@ -114,18 +112,16 @@ markdown==3.3.4 # via tensorboard marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow==3.10.0 +marshmallow==3.11.1 # via # dataclasses-json # marshmallow-enum -matplotlib==3.3.4 - # via -r ../../common/requirements-common.in +matplotlib==3.4.1 + # via -r ../../../common/requirements-common.in mypy-extensions==0.4.3 # via typing-inspect natsort==7.1.1 - # via - # croniter - # flytekit + # via flytekit numpy==1.19.5 # via # h5py @@ -143,17 +139,17 @@ oauthlib==3.1.0 # via requests-oauthlib opt-einsum==3.3.0 # via tensorflow -pandas==1.2.3 +pandas==1.2.4 # via flytekit paramiko==2.7.2 # via sagemaker-training pathspec==0.8.1 # via scantree -pillow==8.1.2 +pillow==8.2.0 # via matplotlib promise==2.3 # via tensorflow-datasets -protobuf==3.15.6 +protobuf==3.15.8 # via # flyteidl # flytekit @@ -194,7 +190,7 @@ pytz==2018.4 # via # flytekit # pandas -regex==2021.3.17 +regex==2021.4.4 # via docker-image-py requests-oauthlib==1.3.0 # via google-auth-oauthlib @@ -205,7 +201,7 @@ requests==2.25.1 # responses # tensorboard # tensorflow-datasets -responses==0.13.1 +responses==0.13.3 # via flytekit retry==0.9.2 # via flytekit @@ -213,13 +209,13 @@ retrying==1.3.3 # via sagemaker-training rsa==4.7.2 # via google-auth -s3transfer==0.3.6 +s3transfer==0.4.2 # via boto3 -sagemaker-training==3.7.3 +sagemaker-training==3.9.2 # via flytekitplugins-awssagemaker scantree==0.0.1 # via dirhash -scipy==1.6.1 +scipy==1.6.3 # via sagemaker-training six==1.15.0 # via @@ -241,7 +237,6 @@ six==1.15.0 # retrying # sagemaker-training # scantree - # tensorboard # tensorflow # tensorflow-datasets sortedcontainers==2.3.0 @@ -250,15 +245,17 @@ statsd==3.3.0 # via flytekit stringcase==1.2.0 # via dataclasses-json +tensorboard-data-server==0.6.1 + # via tensorboard tensorboard-plugin-wit==1.8.0 # via tensorboard -tensorboard==2.4.1 +tensorboard==2.5.0 # via tensorflow tensorflow-datasets==4.2.0 # via -r requirements.in tensorflow-estimator==2.4.0 # via tensorflow -tensorflow-metadata==0.29.0 +tensorflow-metadata==0.30.0 # via tensorflow-datasets tensorflow==2.4.1 # via -r requirements.in @@ -266,7 +263,7 @@ termcolor==1.1.0 # via # tensorflow # tensorflow-datasets -tqdm==4.59.0 +tqdm==4.60.0 # via tensorflow-datasets typing-extensions==3.7.4.3 # via @@ -286,7 +283,7 @@ werkzeug==1.0.1 # tensorboard wheel==0.36.2 # via - # -r ../../common/requirements-common.in + # -r ../../../common/requirements-common.in # astunparse # flytekit # tensorboard @@ -300,7 +297,7 @@ zipp==3.4.1 # via importlib-metadata zope.event==4.5.0 # via gevent -zope.interface==5.3.0 +zope.interface==5.4.0 # via gevent # The following packages are considered to be unsafe in a requirements file: diff --git a/cookbook/integrations/external_services/Makefile b/cookbook/integrations/external_services/Makefile new file mode 100644 index 0000000000..488b3f6b76 --- /dev/null +++ b/cookbook/integrations/external_services/Makefile @@ -0,0 +1,2 @@ +include ../../common/Makefile +include ../../common/parent.mk diff --git a/cookbook/integrations/external_services/hive/Makefile b/cookbook/integrations/external_services/hive/Makefile index 942781c9a8..89c2828290 100644 --- a/cookbook/integrations/external_services/hive/Makefile +++ b/cookbook/integrations/external_services/hive/Makefile @@ -1,5 +1,6 @@ PREFIX=hive include ../../../common/Makefile +include ../../../common/leaf.mk .PHONY: docker_build docker_build: ; diff --git a/cookbook/integrations/external_services/hive/requirements.txt b/cookbook/integrations/external_services/hive/requirements.txt index ed0a633809..f4f0916d54 100644 --- a/cookbook/integrations/external_services/hive/requirements.txt +++ b/cookbook/integrations/external_services/hive/requirements.txt @@ -12,13 +12,13 @@ chardet==4.0.0 # via requests click==7.1.2 # via flytekit -croniter==1.0.8 +croniter==1.0.12 # via flytekit cycler==0.10.0 # via matplotlib -dataclasses-json==0.5.2 +dataclasses-json==0.5.3 # via flytekit -decorator==4.4.2 +decorator==5.0.7 # via retry deprecated==1.2.12 # via flytekit @@ -26,52 +26,48 @@ dirhash==0.2.1 # via flytekit docker-image-py==0.1.10 # via flytekit -flyteidl==0.18.25 +flyteidl==0.18.41 # via flytekit -flytekit==0.17.0b0 +flytekit==0.18.0 # via - # -r ../../common/requirements-common.in + # -r ../../../common/requirements-common.in # flytekitplugins-hive -flytekitplugins-hive==0.16.0 +flytekitplugins-hive==0.18.0 # via -r requirements.in -future==0.18.2 - # via croniter -grpcio==1.36.1 +grpcio==1.37.1 # via flytekit idna==2.10 # via requests -importlib-metadata==3.7.3 +importlib-metadata==4.0.1 # via keyring -keyring==23.0.0 +keyring==23.0.1 # via flytekit kiwisolver==1.3.1 # via matplotlib marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow==3.10.0 +marshmallow==3.11.1 # via # dataclasses-json # marshmallow-enum -matplotlib==3.3.4 - # via -r ../../common/requirements-common.in +matplotlib==3.4.1 + # via -r ../../../common/requirements-common.in mypy-extensions==0.4.3 # via typing-inspect natsort==7.1.1 - # via - # croniter - # flytekit -numpy==1.20.1 + # via flytekit +numpy==1.20.2 # via # matplotlib # pandas # pyarrow -pandas==1.2.3 +pandas==1.2.4 # via flytekit pathspec==0.8.1 # via scantree -pillow==8.1.2 +pillow==8.2.0 # via matplotlib -protobuf==3.15.6 +protobuf==3.15.8 # via # flyteidl # flytekit @@ -93,19 +89,19 @@ pytz==2018.4 # via # flytekit # pandas -regex==2021.3.17 +regex==2021.4.4 # via docker-image-py requests==2.25.1 # via # flytekit # responses -responses==0.13.1 +responses==0.13.3 # via flytekit retry==0.9.2 # via flytekit scantree==0.0.1 # via dirhash -six==1.15.0 +six==1.16.0 # via # cycler # flytekit @@ -120,7 +116,7 @@ statsd==3.3.0 # via flytekit stringcase==1.2.0 # via dataclasses-json -typing-extensions==3.7.4.3 +typing-extensions==3.10.0.0 # via typing-inspect typing-inspect==0.6.0 # via dataclasses-json @@ -131,7 +127,7 @@ urllib3==1.25.11 # responses wheel==0.36.2 # via - # -r ../../common/requirements-common.in + # -r ../../../common/requirements-common.in # flytekit wrapt==1.12.1 # via diff --git a/cookbook/integrations/flytekit_plugins/Makefile b/cookbook/integrations/flytekit_plugins/Makefile new file mode 100644 index 0000000000..488b3f6b76 --- /dev/null +++ b/cookbook/integrations/flytekit_plugins/Makefile @@ -0,0 +1,2 @@ +include ../../common/Makefile +include ../../common/parent.mk diff --git a/cookbook/integrations/flytekit_plugins/pandera/Makefile b/cookbook/integrations/flytekit_plugins/pandera/Makefile index a6cf015dfe..bf84ff61ef 100644 --- a/cookbook/integrations/flytekit_plugins/pandera/Makefile +++ b/cookbook/integrations/flytekit_plugins/pandera/Makefile @@ -1,2 +1,3 @@ PREFIX=pandera include ../../../common/Makefile +include ../../../common/leaf.mk diff --git a/cookbook/integrations/flytekit_plugins/pandera/requirements.in b/cookbook/integrations/flytekit_plugins/pandera/requirements.in index 95c6d76da4..7b5bcd072f 100644 --- a/cookbook/integrations/flytekit_plugins/pandera/requirements.in +++ b/cookbook/integrations/flytekit_plugins/pandera/requirements.in @@ -1,3 +1,3 @@ --r ../../common/requirements-common.in +-r ../../../common/requirements-common.in pandera>=0.6.3 flytekitplugins-pandera>=0.16.0 diff --git a/cookbook/integrations/flytekit_plugins/pandera/requirements.txt b/cookbook/integrations/flytekit_plugins/pandera/requirements.txt index 46d493cb95..a0bff2ee0e 100644 --- a/cookbook/integrations/flytekit_plugins/pandera/requirements.txt +++ b/cookbook/integrations/flytekit_plugins/pandera/requirements.txt @@ -12,13 +12,13 @@ chardet==4.0.0 # via requests click==7.1.2 # via flytekit -croniter==1.0.11 +croniter==1.0.12 # via flytekit cycler==0.10.0 # via matplotlib -dataclasses-json==0.5.2 +dataclasses-json==0.5.3 # via flytekit -decorator==5.0.6 +decorator==5.0.7 # via retry deprecated==1.2.12 # via flytekit @@ -26,19 +26,19 @@ dirhash==0.2.1 # via flytekit docker-image-py==0.1.10 # via flytekit -flyteidl==0.18.32 +flyteidl==0.18.41 # via flytekit -flytekit==0.17.0 +flytekit==0.18.0 # via - # -r ../../common/requirements-common.in + # -r ../../../common/requirements-common.in # flytekitplugins-pandera -flytekitplugins-pandera==0.17.0 +flytekitplugins-pandera==0.18.0 # via -r requirements.in -grpcio==1.37.0 +grpcio==1.37.1 # via flytekit idna==2.10 # via requests -importlib-metadata==3.10.0 +importlib-metadata==4.0.1 # via keyring keyring==23.0.1 # via flytekit @@ -51,7 +51,7 @@ marshmallow==3.11.1 # dataclasses-json # marshmallow-enum matplotlib==3.4.1 - # via -r ../../common/requirements-common.in + # via -r ../../../common/requirements-common.in mypy-extensions==0.4.3 # via typing-inspect natsort==7.1.1 @@ -64,7 +64,7 @@ numpy==1.20.2 # pyarrow packaging==20.9 # via pandera -pandas==1.2.3 +pandas==1.2.4 # via # flytekit # pandera @@ -106,13 +106,13 @@ requests==2.25.1 # via # flytekit # responses -responses==0.13.2 +responses==0.13.3 # via flytekit retry==0.9.2 # via flytekit scantree==0.0.1 # via dirhash -six==1.15.0 +six==1.16.0 # via # cycler # flytekit @@ -127,7 +127,7 @@ statsd==3.3.0 # via flytekit stringcase==1.2.0 # via dataclasses-json -typing-extensions==3.7.4.3 +typing-extensions==3.10.0.0 # via typing-inspect typing-inspect==0.6.0 # via @@ -140,7 +140,7 @@ urllib3==1.25.11 # responses wheel==0.36.2 # via - # -r ../../common/requirements-common.in + # -r ../../../common/requirements-common.in # flytekit wrapt==1.12.1 # via diff --git a/cookbook/integrations/flytekit_plugins/papermilltasks/Makefile b/cookbook/integrations/flytekit_plugins/papermilltasks/Makefile index 152f300428..67ab1bf5ec 100644 --- a/cookbook/integrations/flytekit_plugins/papermilltasks/Makefile +++ b/cookbook/integrations/flytekit_plugins/papermilltasks/Makefile @@ -1,2 +1,3 @@ PREFIX=papermill include ../../../common/Makefile +include ../../../common/leaf.mk diff --git a/cookbook/integrations/flytekit_plugins/papermilltasks/requirements.in b/cookbook/integrations/flytekit_plugins/papermilltasks/requirements.in index dc56a3ba11..ef995a52be 100644 --- a/cookbook/integrations/flytekit_plugins/papermilltasks/requirements.in +++ b/cookbook/integrations/flytekit_plugins/papermilltasks/requirements.in @@ -1,2 +1,2 @@ --r ../../common/requirements-common.in +-r ../../../common/requirements-common.in flytekitplugins-papermill>=0.16.0 diff --git a/cookbook/integrations/flytekit_plugins/papermilltasks/requirements.txt b/cookbook/integrations/flytekit_plugins/papermilltasks/requirements.txt index 5de7e54671..a2f43b9241 100644 --- a/cookbook/integrations/flytekit_plugins/papermilltasks/requirements.txt +++ b/cookbook/integrations/flytekit_plugins/papermilltasks/requirements.txt @@ -20,7 +20,7 @@ attrs==20.3.0 # scantree backcall==0.2.0 # via ipython -black==20.8b1 +black==21.5b0 # via papermill bleach==3.3.0 # via nbconvert @@ -33,13 +33,13 @@ click==7.1.2 # black # flytekit # papermill -croniter==1.0.8 +croniter==1.0.12 # via flytekit cycler==0.10.0 # via matplotlib -dataclasses-json==0.5.2 +dataclasses-json==0.5.3 # via flytekit -decorator==4.4.2 +decorator==5.0.7 # via # ipython # retry @@ -55,29 +55,27 @@ entrypoints==0.3 # via # nbconvert # papermill -flyteidl==0.18.25 +flyteidl==0.18.41 # via flytekit -flytekit==0.17.0b0 +flytekit==0.18.0 # via - # -r ../../common/requirements-common.in + # -r ../../../common/requirements-common.in # flytekitplugins-papermill -flytekitplugins-papermill==0.16.0 +flytekitplugins-papermill==0.18.0 # via -r requirements.in -future==0.18.2 - # via croniter -grpcio==1.36.1 +grpcio==1.37.1 # via flytekit idna==2.10 # via requests -importlib-metadata==3.7.3 +importlib-metadata==4.0.1 # via keyring -ipykernel==5.5.0 +ipykernel==5.5.4 # via flytekitplugins-papermill ipython-genutils==0.2.0 # via # nbformat # traitlets -ipython==7.21.0 +ipython==7.23.1 # via ipykernel jedi==0.18.0 # via ipython @@ -96,7 +94,7 @@ jupyter-core==4.7.1 # nbformat jupyterlab-pygments==0.1.2 # via nbconvert -keyring==23.0.0 +keyring==23.0.1 # via flytekit kiwisolver==1.3.1 # via matplotlib @@ -104,12 +102,14 @@ markupsafe==1.1.1 # via jinja2 marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow==3.10.0 +marshmallow==3.11.1 # via # dataclasses-json # marshmallow-enum -matplotlib==3.3.4 - # via -r ../../common/requirements-common.in +matplotlib-inline==0.1.2 + # via ipython +matplotlib==3.4.1 + # via -r ../../../common/requirements-common.in mistune==0.8.4 # via nbconvert mypy-extensions==0.4.3 @@ -117,36 +117,34 @@ mypy-extensions==0.4.3 # black # typing-inspect natsort==7.1.1 - # via - # croniter - # flytekit + # via flytekit nbclient==0.5.3 # via # nbconvert # papermill nbconvert==6.0.7 # via flytekitplugins-papermill -nbformat==5.1.2 +nbformat==5.1.3 # via # nbclient # nbconvert # papermill nest-asyncio==1.5.1 # via nbclient -numpy==1.20.1 +numpy==1.20.2 # via # matplotlib # pandas # pyarrow packaging==20.9 # via bleach -pandas==1.2.3 +pandas==1.2.4 # via flytekit pandocfilters==1.4.3 # via nbconvert papermill==2.3.3 # via flytekitplugins-papermill -parso==0.8.1 +parso==0.8.2 # via jedi pathspec==0.8.1 # via @@ -156,11 +154,11 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pillow==8.1.2 +pillow==8.2.0 # via matplotlib prompt-toolkit==3.0.18 # via ipython -protobuf==3.15.6 +protobuf==3.15.8 # via # flyteidl # flytekit @@ -170,7 +168,7 @@ py==1.10.0 # via retry pyarrow==3.0.0 # via flytekit -pygments==2.8.1 +pygments==2.9.0 # via # ipython # jupyterlab-pygments @@ -198,7 +196,7 @@ pyyaml==5.4.1 # via papermill pyzmq==22.0.3 # via jupyter-client -regex==2021.3.17 +regex==2021.4.4 # via # black # docker-image-py @@ -207,13 +205,13 @@ requests==2.25.1 # flytekit # papermill # responses -responses==0.13.1 +responses==0.13.3 # via flytekit retry==0.9.2 # via flytekit scantree==0.0.1 # via dirhash -six==1.15.0 +six==1.16.0 # via # bleach # cycler @@ -243,7 +241,7 @@ tornado==6.1 # via # ipykernel # jupyter-client -tqdm==4.59.0 +tqdm==4.60.0 # via papermill traitlets==5.0.5 # via @@ -251,15 +249,12 @@ traitlets==5.0.5 # ipython # jupyter-client # jupyter-core + # matplotlib-inline # nbclient # nbconvert # nbformat -typed-ast==1.4.2 - # via black -typing-extensions==3.7.4.3 - # via - # black - # typing-inspect +typing-extensions==3.10.0.0 + # via typing-inspect typing-inspect==0.6.0 # via dataclasses-json urllib3==1.25.11 @@ -273,7 +268,7 @@ webencodings==0.5.1 # via bleach wheel==0.36.2 # via - # -r ../../common/requirements-common.in + # -r ../../../common/requirements-common.in # flytekit wrapt==1.12.1 # via diff --git a/cookbook/integrations/kubernetes/Makefile b/cookbook/integrations/kubernetes/Makefile new file mode 100644 index 0000000000..488b3f6b76 --- /dev/null +++ b/cookbook/integrations/kubernetes/Makefile @@ -0,0 +1,2 @@ +include ../../common/Makefile +include ../../common/parent.mk diff --git a/cookbook/integrations/kubernetes/k8s_spark/Makefile b/cookbook/integrations/kubernetes/k8s_spark/Makefile index 4a5b0826ee..465300c377 100644 --- a/cookbook/integrations/kubernetes/k8s_spark/Makefile +++ b/cookbook/integrations/kubernetes/k8s_spark/Makefile @@ -1,2 +1,3 @@ PREFIX=k8s_spark include ../../../common/Makefile +include ../../../common/leaf.mk diff --git a/cookbook/integrations/kubernetes/k8s_spark/requirements.txt b/cookbook/integrations/kubernetes/k8s_spark/requirements.txt index d59b9a6d26..9a46d0fdb4 100644 --- a/cookbook/integrations/kubernetes/k8s_spark/requirements.txt +++ b/cookbook/integrations/kubernetes/k8s_spark/requirements.txt @@ -12,13 +12,13 @@ chardet==4.0.0 # via requests click==7.1.2 # via flytekit -croniter==1.0.8 +croniter==1.0.12 # via flytekit cycler==0.10.0 # via matplotlib -dataclasses-json==0.5.2 +dataclasses-json==0.5.3 # via flytekit -decorator==4.4.2 +decorator==5.0.7 # via retry deprecated==1.2.12 # via flytekit @@ -26,52 +26,48 @@ dirhash==0.2.1 # via flytekit docker-image-py==0.1.10 # via flytekit -flyteidl==0.18.25 +flyteidl==0.18.41 # via flytekit -flytekit==0.17.0b0 +flytekit==0.18.0 # via - # -r ../../common/requirements-common.in + # -r ../../../common/requirements-common.in # flytekitplugins-spark -flytekitplugins-spark==0.16.0 +flytekitplugins-spark==0.18.0 # via -r requirements.in -future==0.18.2 - # via croniter -grpcio==1.36.1 +grpcio==1.37.1 # via flytekit idna==2.10 # via requests -importlib-metadata==3.7.3 +importlib-metadata==4.0.1 # via keyring -keyring==23.0.0 +keyring==23.0.1 # via flytekit kiwisolver==1.3.1 # via matplotlib marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow==3.10.0 +marshmallow==3.11.1 # via # dataclasses-json # marshmallow-enum -matplotlib==3.3.4 - # via -r ../../common/requirements-common.in +matplotlib==3.4.1 + # via -r ../../../common/requirements-common.in mypy-extensions==0.4.3 # via typing-inspect natsort==7.1.1 - # via - # croniter - # flytekit -numpy==1.20.1 + # via flytekit +numpy==1.20.2 # via # matplotlib # pandas # pyarrow -pandas==1.2.3 +pandas==1.2.4 # via flytekit pathspec==0.8.1 # via scantree -pillow==8.1.2 +pillow==8.2.0 # via matplotlib -protobuf==3.15.6 +protobuf==3.15.8 # via # flyteidl # flytekit @@ -99,19 +95,19 @@ pytz==2018.4 # via # flytekit # pandas -regex==2021.3.17 +regex==2021.4.4 # via docker-image-py requests==2.25.1 # via # flytekit # responses -responses==0.13.1 +responses==0.13.3 # via flytekit retry==0.9.2 # via flytekit scantree==0.0.1 # via dirhash -six==1.15.0 +six==1.16.0 # via # cycler # flytekit @@ -126,7 +122,7 @@ statsd==3.3.0 # via flytekit stringcase==1.2.0 # via dataclasses-json -typing-extensions==3.7.4.3 +typing-extensions==3.10.0.0 # via typing-inspect typing-inspect==0.6.0 # via dataclasses-json @@ -137,7 +133,7 @@ urllib3==1.25.11 # responses wheel==0.36.2 # via - # -r ../../common/requirements-common.in + # -r ../../../common/requirements-common.in # flytekit wrapt==1.12.1 # via diff --git a/cookbook/integrations/kubernetes/kfpytorch/Makefile b/cookbook/integrations/kubernetes/kfpytorch/Makefile index 8b5910f51d..d5562280ef 100644 --- a/cookbook/integrations/kubernetes/kfpytorch/Makefile +++ b/cookbook/integrations/kubernetes/kfpytorch/Makefile @@ -1,2 +1,3 @@ PREFIX=kfpytorch include ../../../common/Makefile +include ../../../common/leaf.mk diff --git a/cookbook/integrations/kubernetes/pod/Makefile b/cookbook/integrations/kubernetes/pod/Makefile index 3e761280a1..2563c91043 100644 --- a/cookbook/integrations/kubernetes/pod/Makefile +++ b/cookbook/integrations/kubernetes/pod/Makefile @@ -1,2 +1,3 @@ PREFIX=pod include ../../../common/Makefile +include ../../../common/leaf.mk diff --git a/cookbook/integrations/kubernetes/pod/requirements.txt b/cookbook/integrations/kubernetes/pod/requirements.txt index 41f04fdf25..3e56ba2033 100644 --- a/cookbook/integrations/kubernetes/pod/requirements.txt +++ b/cookbook/integrations/kubernetes/pod/requirements.txt @@ -6,7 +6,7 @@ # attrs==20.3.0 # via scantree -cachetools==4.2.1 +cachetools==4.2.2 # via google-auth certifi==2020.12.5 # via @@ -16,13 +16,13 @@ chardet==4.0.0 # via requests click==7.1.2 # via flytekit -croniter==1.0.8 +croniter==1.0.12 # via flytekit cycler==0.10.0 # via matplotlib -dataclasses-json==0.5.2 +dataclasses-json==0.5.3 # via flytekit -decorator==4.4.2 +decorator==5.0.7 # via retry deprecated==1.2.12 # via flytekit @@ -30,25 +30,23 @@ dirhash==0.2.1 # via flytekit docker-image-py==0.1.10 # via flytekit -flyteidl==0.18.25 +flyteidl==0.18.41 # via flytekit -flytekit==0.17.0b0 +flytekit==0.18.0 # via - # -r ../../common/requirements-common.in + # -r ../../../common/requirements-common.in # flytekitplugins-pod -flytekitplugins-pod==0.16.0 +flytekitplugins-pod==0.18.0 # via -r requirements.in -future==0.18.2 - # via croniter -google-auth==1.28.0 +google-auth==1.30.0 # via kubernetes -grpcio==1.36.1 +grpcio==1.37.1 # via flytekit idna==2.10 # via requests -importlib-metadata==3.7.3 +importlib-metadata==4.0.1 # via keyring -keyring==23.0.0 +keyring==23.0.1 # via flytekit kiwisolver==1.3.1 # via matplotlib @@ -56,32 +54,30 @@ kubernetes==12.0.1 # via flytekitplugins-pod marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow==3.10.0 +marshmallow==3.11.1 # via # dataclasses-json # marshmallow-enum -matplotlib==3.3.4 - # via -r ../../common/requirements-common.in +matplotlib==3.4.1 + # via -r ../../../common/requirements-common.in mypy-extensions==0.4.3 # via typing-inspect natsort==7.1.1 - # via - # croniter - # flytekit -numpy==1.20.1 + # via flytekit +numpy==1.20.2 # via # matplotlib # pandas # pyarrow oauthlib==3.1.0 # via requests-oauthlib -pandas==1.2.3 +pandas==1.2.4 # via flytekit pathspec==0.8.1 # via scantree -pillow==8.1.2 +pillow==8.2.0 # via matplotlib -protobuf==3.15.6 +protobuf==3.15.8 # via # flyteidl # flytekit @@ -112,7 +108,7 @@ pytz==2018.4 # pandas pyyaml==5.4.1 # via kubernetes -regex==2021.3.17 +regex==2021.4.4 # via docker-image-py requests-oauthlib==1.3.0 # via kubernetes @@ -122,7 +118,7 @@ requests==2.25.1 # kubernetes # requests-oauthlib # responses -responses==0.13.1 +responses==0.13.3 # via flytekit retry==0.9.2 # via flytekit @@ -130,7 +126,7 @@ rsa==4.7.2 # via google-auth scantree==0.0.1 # via dirhash -six==1.15.0 +six==1.16.0 # via # cycler # flytekit @@ -148,7 +144,7 @@ statsd==3.3.0 # via flytekit stringcase==1.2.0 # via dataclasses-json -typing-extensions==3.7.4.3 +typing-extensions==3.10.0.0 # via typing-inspect typing-inspect==0.6.0 # via dataclasses-json @@ -158,11 +154,11 @@ urllib3==1.25.11 # kubernetes # requests # responses -websocket-client==0.58.0 +websocket-client==0.59.0 # via kubernetes wheel==0.36.2 # via - # -r ../../common/requirements-common.in + # -r ../../../common/requirements-common.in # flytekit wrapt==1.12.1 # via