Skip to content

Commit

Permalink
Unify how docker commands get executed in Breeze (#36131)
Browse files Browse the repository at this point in the history 
So far we executed commands in CI image in breeze in two ways:

* entering the shell (which runs docker-compose under the hood)
* running `docker run` with the CI image

This requires rather complex mapping of environment variables
between `docker-compose` and `docker`. Since recently (#35862)
we can use `shell` command to run commands in very similar way
as docker run (with docker-compose, without database and extra
components - just using the same `breeze shell` mechanisms.

This PR converts all the usages of docker run CI_IMAGE we had
and converts them to use modified `enter_shell` method that has
been moved to "docker_command_utils". This also simplified
passing arguments to the "enter_shell" command - no longer need
to filter out none parameters and **kwargs - all parameters are
passed explicitly.

This also allowed to remove some of the code (extracting args,
filtering_out_none) that are not used anymore.

The entypoint CI has been slightly refactored - to provide
a bit better structure and handle `--skip-environment-initialization`
better - we can now both set `--use-airflow-version` and
`--skip-environment-initialization` which was not possible
before.

(cherry picked from commit 2d15dbf)
  • Loading branch information
@potiuk
potiuk committed Dec 15, 2023
1 parent d3e6283 commit 2bc7c40
Show file tree
Hide file tree
Showing 28 changed files with 675 additions and 785 deletions.
122 changes: 67 additions & 55 deletions Dockerfile.ci
View file
Original file line number Diff line number Diff line change
Expand Up @@ -710,7 +710,7 @@ PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.8}

export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}}

: "${AIRFLOW_SOURCES:?"ERROR: AIRFLOW_SOURCES not set !!!!"}"
mkdir "${AIRFLOW_HOME}/sqlite" -p || true

ASSET_COMPILATION_WAIT_MULTIPLIER=${ASSET_COMPILATION_WAIT_MULTIPLIER:=1}

Expand Down Expand Up @@ -755,8 +755,10 @@ If it does not complete soon, you might want to stop it and remove file lock:
fi
}

if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then

function environment_initialization() {
if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} == "true" ]]; then
return
fi
if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
if [[ ${BACKEND:=} == "mssql" ]]; then
echo "${COLOR_RED}ARM platform is not supported for ${BACKEND} backend. Exiting.${COLOR_RESET}"
Expand All @@ -778,6 +780,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
echo
export AIRFLOW__SCHEDULER__STANDALONE_DAG_PROCESSOR=True
fi

if [[ ${DATABASE_ISOLATION=} == "true" ]]; then
echo "${COLOR_BLUE}Force database isolation configuration:${COLOR_RESET}"
export AIRFLOW__CORE__DATABASE_ACCESS_ISOLATION=True
Expand All @@ -787,33 +790,10 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then

RUN_TESTS=${RUN_TESTS:="false"}
CI=${CI:="false"}
USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION:=""}"

if [[ ${USE_AIRFLOW_VERSION} == "" && ${USE_PACKAGES_FROM_DIST=} != "true" ]]; then
export PYTHONPATH=${AIRFLOW_SOURCES}
echo
echo "${COLOR_BLUE}Using airflow version from current sources${COLOR_RESET}"
echo
# Cleanup the logs, tmp when entering the environment
sudo rm -rf "${AIRFLOW_SOURCES}"/logs/*
sudo rm -rf "${AIRFLOW_SOURCES}"/tmp/*
mkdir -p "${AIRFLOW_SOURCES}"/logs/
mkdir -p "${AIRFLOW_SOURCES}"/tmp/
else
python "${IN_CONTAINER_DIR}/install_airflow_and_providers.py"
fi

if [[ "${USE_AIRFLOW_VERSION}" =~ ^2\.2\..*|^2\.1\..*|^2\.0\..* && "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=}" != "" ]]; then
# make sure old variable is used for older airflow versions
export AIRFLOW__CORE__SQL_ALCHEMY_CONN="${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN}"
fi

# Added to have run-tests on path
export PATH=${PATH}:${AIRFLOW_SOURCES}

# This is now set in conftest.py - only for pytest tests
unset AIRFLOW__CORE__UNIT_TEST_MODE

mkdir -pv "${AIRFLOW_HOME}/logs/"

# Change the default worker_concurrency for tests
Expand Down Expand Up @@ -869,54 +849,86 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
# shellcheck source=scripts/in_container/bin/run_tmux
exec run_tmux
fi
fi
}

function determine_airflow_to_use() {
USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION:=""}"
if [[ ${USE_AIRFLOW_VERSION} == "" && ${USE_PACKAGES_FROM_DIST=} != "true" ]]; then
export PYTHONPATH=${AIRFLOW_SOURCES}
echo
echo "${COLOR_BLUE}Using airflow version from current sources${COLOR_RESET}"
echo
# Cleanup the logs, tmp when entering the environment
sudo rm -rf "${AIRFLOW_SOURCES}"/logs/*
sudo rm -rf "${AIRFLOW_SOURCES}"/tmp/*
mkdir -p "${AIRFLOW_SOURCES}"/logs/
mkdir -p "${AIRFLOW_SOURCES}"/tmp/
else
python "${IN_CONTAINER_DIR}/install_airflow_and_providers.py"
fi

rm -f "${AIRFLOW_SOURCES}/pytest.ini"
rm -f "${AIRFLOW_SOURCES}/.coveragerc"
if [[ "${USE_AIRFLOW_VERSION}" =~ ^2\.2\..*|^2\.1\..*|^2\.0\..* && "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=}" != "" ]]; then
# make sure old variable is used for older airflow versions
export AIRFLOW__CORE__SQL_ALCHEMY_CONN="${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN}"
fi
}

if [[ ${UPGRADE_BOTO=} == "true" ]]; then
function check_boto_upgrade() {
if [[ ${UPGRADE_BOTO=} != "true" ]]; then
return
fi
echo
echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
echo
pip uninstall --root-user-action ignore aiobotocore s3fs -y || true
pip install --root-user-action ignore --upgrade boto3 botocore
pip check
fi
if [[ ${DOWNGRADE_SQLALCHEMY=} == "true" ]]; then
}

function check_download_sqlalchemy() {
if [[ ${DOWNGRADE_SQLALCHEMY=} != "true" ]]; then
return
fi
min_sqlalchemy_version=$(grep "sqlalchemy>=" setup.cfg | sed "s/.*>=\([0-9\.]*\).*/\1/")
echo
echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}"
echo
pip install --root-user-action ignore "sqlalchemy==${min_sqlalchemy_version}"
pip check
fi
}

pip uninstall --root-user-action ignore "pytest-capture-warnings" -y >/dev/null 2>&1 || true
function check_run_tests() {
if [[ ${RUN_TESTS=} != "true" ]]; then
return
fi

set +u
if [[ "${RUN_TESTS}" != "true" ]]; then
exec /bin/bash "${@}"
fi
set -u
if [[ ${REMOVE_ARM_PACKAGES:="false"} == "true" ]]; then
# Test what happens if we do not have ARM packages installed.
# This is useful to see if pytest collection works without ARM packages which is important
# for the MacOS M1 users running tests in their ARM machines with `breeze testing tests` command
python "${IN_CONTAINER_DIR}/remove_arm_packages.py"
fi

if [[ ${REMOVE_ARM_PACKAGES:="false"} == "true" ]]; then
# Test what happens if we do not have ARM packages installed.
# This is useful to see if pytest collection works without ARM packages which is important
# for the MacOS M1 users running tests in their ARM machines with `breeze testing tests` command
python "${IN_CONTAINER_DIR}/remove_arm_packages.py"
fi
if [[ ${TEST_TYPE} == "PlainAsserts" ]]; then
# Plain asserts should be converted to env variable to make sure they are taken into account
# otherwise they will not be effective during test collection when plain assert is breaking collection
export PYTEST_PLAIN_ASSERTS="true"
fi

if [[ ${TEST_TYPE} == "PlainAsserts" ]]; then
# Plain asserts should be converted to env variable to make sure they are taken into account
# otherwise they will not be effective during test collection when plain assert is breaking collection
export PYTEST_PLAIN_ASSERTS="true"
fi
if [[ ${RUN_SYSTEM_TESTS:="false"} == "true" ]]; then
exec "${IN_CONTAINER_DIR}/run_system_tests.sh" "${@}"
else
exec "${IN_CONTAINER_DIR}/run_ci_tests.sh" "${@}"
fi
}

if [[ ${RUN_SYSTEM_TESTS:="false"} == "true" ]]; then
"${IN_CONTAINER_DIR}/run_system_tests.sh" "${@}"
else
"${IN_CONTAINER_DIR}/run_ci_tests.sh" "${@}"
fi
determine_airflow_to_use
environment_initialization
check_boto_upgrade
check_download_sqlalchemy
check_run_tests "${@}"

exec /bin/bash "${@}"
EOF

# The content below is automatically copied from scripts/docker/entrypoint_exec.sh
Expand Down
Loading

0 comments on commit 2bc7c40

Please sign in to comment.