From 7ddc9bc2ab993316dc9871319797c7ecd2b5aa26 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 15 Apr 2024 16:08:48 +0200 Subject: [PATCH] Fix PROD image package installation in CI When PROD image packages are installed in in CI, the local sources should not be present in the image, also constraints from sources shoudl replace the one downloaded from main. --- Dockerfile | 4 ++++ .../src/airflow_breeze/params/build_prod_params.py | 13 +++++++++++++ docker_tests/test_prod_image.py | 6 ++++++ scripts/docker/install_from_docker_context_files.sh | 4 ++++ 4 files changed, 27 insertions(+) diff --git a/Dockerfile b/Dockerfile index 94798fbdb7951..9ae60c52bd1dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -780,6 +780,10 @@ function install_airflow_and_providers_from_docker_context_files(){ ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint "${local_constraints_file}" \ "${install_airflow_package[@]}" "${installing_providers_packages[@]}" set +x + echo + echo "${COLOR_BLUE}Copying ${local_constraints_file} to ${HOME}/constraints.txt${COLOR_RESET}" + echo + cp "${local_constraints_file}" "${HOME}/constraints.txt" else echo echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}" diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index d3a3dbfdbc35a..d6193963ce4ac 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -143,6 +143,19 @@ def _extra_prod_docker_build_flags(self) -> list[str]: ) self.airflow_constraints_location = constraints_location extra_build_flags.extend(self.args_for_remote_install) + elif self.install_packages_from_context: + extra_build_flags.extend( + [ + "--build-arg", + "AIRFLOW_SOURCES_FROM=/empty", + "--build-arg", + "AIRFLOW_SOURCES_TO=/empty", + "--build-arg", + f"AIRFLOW_INSTALLATION_METHOD={self.installation_method}", + "--build-arg", + f"AIRFLOW_CONSTRAINTS_REFERENCE={self.airflow_constraints_reference}", + ], + ) else: extra_build_flags.extend( [ diff --git a/docker_tests/test_prod_image.py b/docker_tests/test_prod_image.py index ab35c63bffa53..1d7f043b85df7 100644 --- a/docker_tests/test_prod_image.py +++ b/docker_tests/test_prod_image.py @@ -175,6 +175,12 @@ def test_pip_dependencies_conflict(self, default_docker_image): def test_check_dependencies_imports(self, package_name, import_names, default_docker_image): run_python_in_docker(f"import {','.join(import_names)}", image=default_docker_image) + def test_there_is_no_opt_airflow_airflow_folder(self, default_docker_image): + output = run_bash_in_docker( + "find /opt/airflow/airflow/ 2>/dev/null | wc -l", image=default_docker_image + ) + assert output == "0" + class TestExecuteAsRoot: def test_execute_airflow_as_root(self, default_docker_image): diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index d6fab1e8273ce..edcb50c82e054 100644 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -86,6 +86,10 @@ function install_airflow_and_providers_from_docker_context_files(){ ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint "${local_constraints_file}" \ "${install_airflow_package[@]}" "${installing_providers_packages[@]}" set +x + echo + echo "${COLOR_BLUE}Copying ${local_constraints_file} to ${HOME}/constraints.txt${COLOR_RESET}" + echo + cp "${local_constraints_file}" "${HOME}/constraints.txt" else echo echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}"