Fixed builds for FlyteSnacks with improved Makefile system to allow u…

…ses one set of commands (flyteorg#196)

boilerplate/flyte/update.sh

@@ -11,4 +11,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 # Clone the config.yml file
 echo "     - copying ${DIR}/config.yml to the root directory."
-cp ${DIR}/config.yml ${DIR}/../../.github/config.yml
+cp "${DIR}"/config.yml "${DIR}"/../../.github/config.yml

boilerplate/update.sh

@@ -10,7 +10,7 @@ set -e
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 OUT="$(mktemp -d)"
-trap "rm -fr $OUT" EXIT
+trap 'rm -fr $OUT' EXIT
 
 git clone git@github.com:flyteorg/boilerplate.git "${OUT}"
 
@@ -30,12 +30,12 @@ if [ ! -f "$CONFIG_FILE" ]; then
 fi
 
 if [ -z "$REPOSITORY" ]; then
-  echo '$REPOSITORY is required to run this script'
+  echo "$REPOSITORY is required to run this script"
   echo "See $README for more details."
   exit 1
 fi
 
-while read directory junk; do
+while read -r directory junk; do
   # Skip comment lines (which can have leading whitespace)
   if [[ "$directory" == '#'* ]]; then
     continue
@@ -63,13 +63,13 @@ while read directory junk; do
   echo "$directory is configured in update.cfg."
   echo "-----------------------------------------------------------------------------------"
   echo "syncing files from source."
-  rm -rf "${DIR}/${directory}"
-  mkdir -p $(dirname "${DIR}/${directory}")
+  rm -rf "${DIR:?}/${directory}"
+  mkdir -p "$(dirname "${DIR}"/"${directory}")"
   cp -r "$dir_path" "${DIR}/${directory}"
   if [ -f "${DIR}/${directory}/update.sh" ]; then
     echo "executing ${DIR}/${directory}/update.sh"
     "${DIR}/${directory}/update.sh"
   fi
   echo "***********************************************************************************"
   echo ""
-done < "$CONFIG_FILE"
+done < "$CONFIG_FILE"

cookbook/Makefile

@@ -1,6 +1,7 @@
 .SILENT:
 
 include common/Makefile
+include common/parent.mk
 
 dev-requirements.txt: export CUSTOM_COMPILE_COMMAND := $(MAKE) dev-requirements.txt
 dev-requirements.txt: dev-requirements.in install-piptools
@@ -24,55 +25,6 @@ enter_sandbox: docker_build
 run-examples: ## Runs all examples
 	sh scripts/run-all-examples.sh
 
-.PHONY: all_fast_register
-all_fast_register: ## Registers new code changes using the last built image (assumes current HEAD refers to a built image).
-	$(MAKE) -C core fast_register
-	$(MAKE) -C plugins all_fast_register
-	$(MAKE) -C case_studies all_fast_register
-
-.PHONY: all_fast_serialize
-all_fast_serialize:
-	$(MAKE) -C core fast_serialize
-	$(MAKE) -C plugins all_fast_serialize
-	$(MAKE) -C case_studies all_fast_serialize
-
-.PHONY: all_requirements
-all_requirements: dev-requirements docs-requirements ## Makes all requirements in sub directories
-	$(MAKE) -C core requirements
-	$(MAKE) -C plugins all_requirements
-	$(MAKE) -C case_studies all_requirements
-
-.PHONY: all_register
-all_register: ## Builds, pushes and registers all docker images, workflows and tasks in all sub directories.
-	$(MAKE) -j10 -C core register
-	$(MAKE) -j10 -C plugins all_register
-	$(MAKE) -j10 -C case_studies all_register
-
-.PHONY: all_serialize
-all_serialize: # Builds and serializes all docker images, workflows and tasks in all sub directories.
-	$(MAKE) -C core serialize
-	$(MAKE) -C plugins all_serialize
-	$(MAKE) -C case_studies all_serialize
-
-.PHONY: all_docker_push
-all_docker_push: # Builds and pushes all docker images.
-	$(MAKE) -j10 -C core docker_push
-	$(MAKE) -j10 -C plugins all_docker_push
-	$(MAKE) -j10 -C case_studies all_docker_push
-
-.PHONY: all_k3d_load_image
-all_k3d_load_image: 
-	$(MAKE) -j10 -C core k3d_load_image
-	$(MAKE) -j10 -C plugins all_k3d_load_image
-	$(MAKE) -j10 -C case_studies all_k3d_load_image
-
-.PHONY: all_clean
-all_clean: # Cleans build directories (e.g. _pb_output/)
-	$(MAKE) -C core clean
-	$(MAKE) -C docs clean
-	$(MAKE) -C plugins all_clean
-	$(MAKE) -C case_studies all_clean
-
 .PHONY: archive
 archive: # archive directories (e.g. _pb_output/)
-	sh scripts/archive-all-example.sh
+	sh scripts/archive-all-example.sh

cookbook/case_studies/house_price_prediction/Makefile

@@ -1,2 +1,3 @@
 PREFIX=house_price_prediction
 include ../../common/Makefile
+include ../../common/leaf.mk

cookbook/case_studies/house_price_prediction/requirements.txt

@@ -12,71 +12,67 @@ chardet==4.0.0
     # via requests
 click==7.1.2
     # via flytekit
-croniter==1.0.8
+croniter==1.0.12
     # via flytekit
 cycler==0.10.0
     # via matplotlib
-dataclasses-json==0.5.2
+dataclasses-json==0.5.3
     # via flytekit
-decorator==4.4.2
+decorator==5.0.7
     # via retry
 deprecated==1.2.12
     # via flytekit
 dirhash==0.2.1
     # via flytekit
 docker-image-py==0.1.10
     # via flytekit
-flyteidl==0.18.25
+flyteidl==0.18.41
     # via flytekit
-flytekit==0.17.0b1
+flytekit==0.18.0
     # via -r ../../common/requirements-common.in
-future==0.18.2
-    # via croniter
-grpcio==1.36.1
+grpcio==1.37.1
     # via flytekit
 idna==2.10
     # via requests
-importlib-metadata==3.7.3
+importlib-metadata==4.0.1
     # via keyring
 joblib==1.0.1
     # via
     #   -r requirements.in
     #   scikit-learn
-keyring==23.0.0
+keyring==23.0.1
     # via flytekit
 kiwisolver==1.3.1
     # via matplotlib
 marshmallow-enum==1.5.1
     # via dataclasses-json
-marshmallow==3.10.0
+marshmallow==3.11.1
     # via
     #   dataclasses-json
     #   marshmallow-enum
-matplotlib==3.3.4
+matplotlib==3.4.1
     # via
     #   -r ../../common/requirements-common.in
     #   -r requirements.in
 mypy-extensions==0.4.3
     # via typing-inspect
 natsort==7.1.1
-    # via
-    #   croniter
-    #   flytekit
-numpy==1.20.1
+    # via flytekit
+numpy==1.20.2
     # via
     #   matplotlib
     #   pandas
     #   pyarrow
     #   scikit-learn
     #   scipy
     #   xgboost
-pandas==1.2.3
+pandas==1.2.4
     # via flytekit
 pathspec==0.8.1
     # via scantree
-pillow==8.1.2
+pillow==8.2.0
     # via matplotlib
-protobuf==3.15.6
+protobuf==3.15.8
     # via
     #   flyteidl
     #   flytekit
@@ -98,25 +94,25 @@ pytz==2018.4
     # via
     #   flytekit
     #   pandas
-regex==2021.3.17
+regex==2021.4.4
     # via docker-image-py
 requests==2.25.1
     # via
     #   flytekit
     #   responses
-responses==0.13.1
+responses==0.13.3
     # via flytekit
 retry==0.9.2
     # via flytekit
 scantree==0.0.1
     # via dirhash
-scikit-learn==0.24.1
+scikit-learn==0.24.2
     # via sklearn
-scipy==1.6.1
+scipy==1.6.3
     # via
     #   scikit-learn
     #   xgboost
-six==1.15.0
+six==1.16.0
     # via
     #   cycler
     #   flytekit
@@ -137,7 +133,7 @@ tabulate==0.8.9
     # via -r requirements.in
 threadpoolctl==2.1.0
     # via scikit-learn
-typing-extensions==3.7.4.3
+typing-extensions==3.10.0.0
     # via typing-inspect
 typing-inspect==0.6.0
     # via dataclasses-json
@@ -154,7 +150,7 @@ wrapt==1.12.1
     # via
     #   deprecated
     #   flytekit
-xgboost==1.3.3
+xgboost==1.4.1
     # via -r requirements.in
 zipp==3.4.1
     # via importlib-metadata

cookbook/case_studies/ml_training/Makefile

@@ -0,0 +1,2 @@
+include ../common/Makefile
+include ../common/parent.mk

cookbook/case_studies/ml_training/house_price_prediction/Makefile

@@ -1,2 +1,3 @@
 PREFIX=house_price_prediction
 include ../../../common/Makefile
+include ../../../common/leaf.mk

cookbook/case_studies/ml_training/house_price_prediction/requirements.in

@@ -1,4 +1,4 @@
--r ../../common/requirements-common.in
+-r ../../../common/requirements-common.in
 xgboost
 joblib
 sklearn

cookbook/case_studies/ml_training/pima_diabetes/Makefile

@@ -1,2 +1,3 @@
 PREFIX=pima_diabetes
 include ../../../common/Makefile
+include ../../../common/leaf.mk