diff --git a/Dockerfile b/Dockerfile index d406230d91a7e..333802f083867 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1411,7 +1411,7 @@ function install_yarn_dependencies_from_branch_tip() { local TEMP_AIRFLOW_DIR TEMP_AIRFLOW_DIR=$(mktemp -d) # Download the source code from the specified branch - set +e + set -e set -x curl -fsSL "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" | \ tar xz -C "${TEMP_AIRFLOW_DIR}" --strip 1 @@ -1419,14 +1419,59 @@ function install_yarn_dependencies_from_branch_tip() { cd "${TEMP_AIRFLOW_DIR}/airflow/www" yarn install --frozen-lockfile set +x - set -e + set +e echo "${COLOR_BLUE}Yarn dependencies installed successfully${COLOR_RESET}" + + # Copy Yarn packages to the .yarn-cache directory + echo + echo "${COLOR_BLUE}Copying Yarn packages to the ${YARN_CACHE_DIR} directory${COLOR_RESET}" + echo + set -e + set -x + cp -r ./node_modules $YARN_CACHE_DIR/ + echo "${COLOR_BLUE}Yarn packages copied successfully${COLOR_RESET}" + set +x # Clean up + remove_npm_and_yarn rm -rf "${TEMP_AIRFLOW_DIR}" + set +e +} + +function install_npm_and_yarn() { + echo + echo "${COLOR_BLUE}Installing npm and yarn${COLOR_RESET}" + echo + set -e + set -x + # Install npm + apt-get update + apt-get install -y npm + # Install yarn + npm install -g yarn + echo "${COLOR_BLUE}npm and yarn installed successfully${COLOR_RESET}" + set +x + set +e +} + +function remove_npm_and_yarn() { + echo + echo "${COLOR_BLUE}Removing npm and yarn${COLOR_RESET}" + echo + set +e + set -x + # Remove yarn + npm cache clean --force + npm uninstall -g yarn + # Remove npm + apt-get remove -y npm + echo "${COLOR_BLUE}npm and yarn removed successfully${COLOR_RESET}" + set +x + set -e } common::get_colors +install_npm_and_yarn install_yarn_dependencies_from_branch_tip EOF @@ -1488,6 +1533,8 @@ ARG DOCKER_CONTEXT_FILES="Dockerfile" ARG AIRFLOW_HOME ARG AIRFLOW_USER_HOME_DIR ARG AIRFLOW_UID +ARG AIRFLOW_REPO=apache/airflow +ARG AIRFLOW_BRANCH=main RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \ --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" && \ @@ -1495,6 +1542,19 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw COPY --chown=${AIRFLOW_UID}:0 ${DOCKER_CONTEXT_FILES} /docker-context-files +# Yarn cache +ARG AIRFLOW_PRE_CACHED_YARN_PACKAGES="true" +ENV AIRFLOW_PRE_CACHED_YARN_PACKAGES=${AIRFLOW_PRE_CACHED_YARN_PACKAGES} +COPY --from=scripts install_yarn_dependencies_from_branch_tip.sh /scripts/docker/ + +ENV YARN_CACHE_DIR="${AIRFLOW_USER_HOME_DIR}/.yarn-cache" +RUN mkdir -p "${YARN_CACHE_DIR}" && \ + chown -R "airflow:0" "${YARN_CACHE_DIR}" +# We are installing Yarn dependencies here to make sure they are cached in the layer +RUN if [[ ${AIRFLOW_PRE_CACHED_YARN_PACKAGES} == "true" ]]; then \ + bash /scripts/docker/install_yarn_dependencies_from_branch_tip.sh; \ + fi + USER airflow ARG AIRFLOW_REPO=apache/airflow @@ -1513,8 +1573,6 @@ ARG PIP_PROGRESS_BAR # By default we do not use pre-cached packages, but in CI/Breeze environment we override this to speed up # builds in case pyproject.toml changed. This is pure optimisation of CI/Breeze builds. ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="false" -# By default we do not use pre-cached yarn packages, but in CI/Breeze environment we override this to speed up -ARG AIRFLOW_PRE_CACHED_YARN_PACKAGES="true" # This is airflow version that is put in the label of the image build ARG AIRFLOW_VERSION # By default latest released version of airflow is installed (when empty) but this value can be overridden @@ -1553,7 +1611,6 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ UV_HTTP_TIMEOUT=${UV_HTTP_TIMEOUT} \ AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \ AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ - AIRFLOW_PRE_CACHED_YARN_PACKAGES=${AIRFLOW_PRE_CACHED_YARN_PACKAGES} \ AIRFLOW_VERSION=${AIRFLOW_VERSION} \ AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} \ AIRFLOW_VERSION_SPECIFICATION=${AIRFLOW_VERSION_SPECIFICATION} \ @@ -1579,8 +1636,7 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ # Copy all scripts required for installation - changing any of those should lead to # rebuilding from here COPY --from=scripts common.sh install_packaging_tools.sh \ - install_airflow_dependencies_from_branch_tip.sh create_prod_venv.sh \ - install_yarn_dependencies_from_branch_tip.sh /scripts/docker/ + install_airflow_dependencies_from_branch_tip.sh create_prod_venv.sh /scripts/docker/ # We can set this value to true in case we want to install .whl/.tar.gz packages placed in the # docker-context-files folder. This can be done for both additional packages you want to install @@ -1618,12 +1674,6 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \ bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi -# We are installing Yarn dependencies here to make sure they are cached in the layer -RUN if [[ ${AIRFLOW_PRE_CACHED_YARN_PACKAGES} == "true" ]]; then \ - bash /scripts/docker/install_yarn_dependencies_from_branch_tip.sh; \ - fi - - COPY --chown=airflow:0 ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO} # Add extra python dependencies diff --git a/Dockerfile.ci b/Dockerfile.ci index 0a42d15e87603..1dfe69866d3c8 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1186,7 +1186,7 @@ function install_yarn_dependencies_from_branch_tip() { local TEMP_AIRFLOW_DIR TEMP_AIRFLOW_DIR=$(mktemp -d) # Download the source code from the specified branch - set +e + set -e set -x curl -fsSL "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" | \ tar xz -C "${TEMP_AIRFLOW_DIR}" --strip 1 @@ -1194,14 +1194,59 @@ function install_yarn_dependencies_from_branch_tip() { cd "${TEMP_AIRFLOW_DIR}/airflow/www" yarn install --frozen-lockfile set +x - set -e + set +e echo "${COLOR_BLUE}Yarn dependencies installed successfully${COLOR_RESET}" + + # Copy Yarn packages to the .yarn-cache directory + echo + echo "${COLOR_BLUE}Copying Yarn packages to the ${YARN_CACHE_DIR} directory${COLOR_RESET}" + echo + set -e + set -x + cp -r ./node_modules $YARN_CACHE_DIR/ + echo "${COLOR_BLUE}Yarn packages copied successfully${COLOR_RESET}" + set +x # Clean up + remove_npm_and_yarn rm -rf "${TEMP_AIRFLOW_DIR}" + set +e +} + +function install_npm_and_yarn() { + echo + echo "${COLOR_BLUE}Installing npm and yarn${COLOR_RESET}" + echo + set -e + set -x + # Install npm + apt-get update + apt-get install -y npm + # Install yarn + npm install -g yarn + echo "${COLOR_BLUE}npm and yarn installed successfully${COLOR_RESET}" + set +x + set +e +} + +function remove_npm_and_yarn() { + echo + echo "${COLOR_BLUE}Removing npm and yarn${COLOR_RESET}" + echo + set +e + set -x + # Remove yarn + npm cache clean --force + npm uninstall -g yarn + # Remove npm + apt-get remove -y npm + echo "${COLOR_BLUE}npm and yarn removed successfully${COLOR_RESET}" + set +x + set -e } common::get_colors +install_npm_and_yarn install_yarn_dependencies_from_branch_tip EOF @@ -1311,7 +1356,6 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" # It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH environment variable. ARG AIRFLOW_CI_BUILD_EPOCH="10" ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" -ARG AIRFLOW_PRE_CACHED_YARN_PACKAGES="true" # Setup PIP # By default PIP install run without cache to make image smaller ARG PIP_NO_CACHE_DIR="true" @@ -1345,7 +1389,6 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} \ AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ - AIRFLOW_PRE_CACHED_YARN_PACKAGES=${AIRFLOW_PRE_CACHED_YARN_PACKAGES} \ AIRFLOW_VERSION=${AIRFLOW_VERSION} \ AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ @@ -1387,11 +1430,20 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \ bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi +# Yarn cache +ARG AIRFLOW_PRE_CACHED_YARN_PACKAGES="true" +ENV AIRFLOW_PRE_CACHED_YARN_PACKAGES=${AIRFLOW_PRE_CACHED_YARN_PACKAGES} +COPY --from=scripts install_yarn_dependencies_from_branch_tip.sh /scripts/docker/ + +ENV YARN_CACHE_DIR="${AIRFLOW_USER_HOME_DIR}/.yarn-cache" +RUN mkdir -p "${YARN_CACHE_DIR}" && \ + chown -R "airflow:0" "${YARN_CACHE_DIR}" # We are installing Yarn dependencies here to make sure they are cached in the layer RUN if [[ ${AIRFLOW_PRE_CACHED_YARN_PACKAGES} == "true" ]]; then \ bash /scripts/docker/install_yarn_dependencies_from_branch_tip.sh; \ fi + # Here we fix the versions so all subsequent commands will use the versions # from the sources # You can swap comments between those two args to test pip from the main version diff --git a/scripts/ci/pre_commit/compile_www_assets.py b/scripts/ci/pre_commit/compile_www_assets.py index 6a517860d8116..4367354ac31f1 100755 --- a/scripts/ci/pre_commit/compile_www_assets.py +++ b/scripts/ci/pre_commit/compile_www_assets.py @@ -56,6 +56,7 @@ def get_directory_hash(directory: Path, skip_path_regexp: str | None = None) -> www_directory = AIRFLOW_SOURCES_PATH / "airflow" / "www" node_modules_directory = www_directory / "node_modules" dist_directory = www_directory / "static" / "dist" + YARN_CACHE_DIR = os.getenv("YARN_CACHE_DIR", f"{Path.home()}/.yarn-cache") WWW_HASH_FILE.parent.mkdir(exist_ok=True, parents=True) if node_modules_directory.exists() and dist_directory.exists(): old_hash = WWW_HASH_FILE.read_text() if WWW_HASH_FILE.exists() else "" @@ -68,7 +69,12 @@ def get_directory_hash(directory: Path, skip_path_regexp: str | None = None) -> shutil.rmtree(dist_directory, ignore_errors=True) env = os.environ.copy() env["FORCE_COLOR"] = "true" - if os.getenv("AIRFLOW_PRE_CACHED_YARN_PACKAGES") != "true": + if os.getenv("AIRFLOW_PRE_CACHED_YARN_PACKAGES") == "true": + # Copy yarn-cache to node_modules from yarn-cache + shutil.copytree(YARN_CACHE_DIR, www_directory) + # Remove the yarn-cache directory to reduce the size of the docker image and prevent duplication + shutil.rmtree(YARN_CACHE_DIR, ignore_errors=True) + else: subprocess.check_call(["yarn", "install", "--frozen-lockfile"], cwd=os.fspath(www_directory)) subprocess.check_call(["yarn", "run", "build"], cwd=os.fspath(www_directory), env=env) new_hash = get_directory_hash(www_directory, skip_path_regexp=r".*node_modules.*") diff --git a/scripts/docker/install_yarn_dependencies_from_branch_tip.sh b/scripts/docker/install_yarn_dependencies_from_branch_tip.sh index 3adcf695b2128..309e67d018422 100644 --- a/scripts/docker/install_yarn_dependencies_from_branch_tip.sh +++ b/scripts/docker/install_yarn_dependencies_from_branch_tip.sh @@ -37,7 +37,7 @@ function install_yarn_dependencies_from_branch_tip() { local TEMP_AIRFLOW_DIR TEMP_AIRFLOW_DIR=$(mktemp -d) # Download the source code from the specified branch - set +e + set -e set -x curl -fsSL "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" | \ tar xz -C "${TEMP_AIRFLOW_DIR}" --strip 1 @@ -45,12 +45,59 @@ function install_yarn_dependencies_from_branch_tip() { cd "${TEMP_AIRFLOW_DIR}/airflow/www" yarn install --frozen-lockfile set +x - set -e + set +e echo "${COLOR_BLUE}Yarn dependencies installed successfully${COLOR_RESET}" + + # Copy Yarn packages to the .yarn-cache directory + echo + echo "${COLOR_BLUE}Copying Yarn packages to the ${YARN_CACHE_DIR} directory${COLOR_RESET}" + echo + set -e + set -x + cp -r ./node_modules $YARN_CACHE_DIR/ + echo "${COLOR_BLUE}Yarn packages copied successfully${COLOR_RESET}" + set +x # Clean up + remove_npm_and_yarn rm -rf "${TEMP_AIRFLOW_DIR}" + set +e +} + +# Install npm and yarn function +function install_npm_and_yarn() { + echo + echo "${COLOR_BLUE}Installing npm and yarn${COLOR_RESET}" + echo + set -e + set -x + # Install npm + apt-get update + apt-get install -y npm + # Install yarn + npm install -g yarn + echo "${COLOR_BLUE}npm and yarn installed successfully${COLOR_RESET}" + set +x + set +e +} + +# Remove yarn and npm function +function remove_npm_and_yarn() { + echo + echo "${COLOR_BLUE}Removing npm and yarn${COLOR_RESET}" + echo + set +e + set -x + # Remove yarn + npm cache clean --force + npm uninstall -g yarn + # Remove npm + apt-get remove -y npm + echo "${COLOR_BLUE}npm and yarn removed successfully${COLOR_RESET}" + set +x + set -e } common::get_colors +install_npm_and_yarn install_yarn_dependencies_from_branch_tip