Fix -+e for proper error catching, install/remove yarn and npm, copy …

…node_modules and remove when utilised
apache · Nov 16, 2024 · 38c7f3e · 38c7f3e
1 parent 9df949a
commit 38c7f3e
Show file tree

Hide file tree

Showing 4 changed files with 175 additions and 20 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1411,22 +1411,67 @@ function install_yarn_dependencies_from_branch_tip() {
     local TEMP_AIRFLOW_DIR
     TEMP_AIRFLOW_DIR=$(mktemp -d)
     # Download the source code from the specified branch
-    set +e
+    set -e
     set -x
     curl -fsSL "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" | \
         tar xz -C "${TEMP_AIRFLOW_DIR}" --strip 1
     # Install Yarn dependencies
     cd "${TEMP_AIRFLOW_DIR}/airflow/www"
     yarn install --frozen-lockfile
     set +x
-    set -e
+    set +e
     echo "${COLOR_BLUE}Yarn dependencies installed successfully${COLOR_RESET}"
+
+    # Copy Yarn packages to the .yarn-cache directory
+    echo
+    echo "${COLOR_BLUE}Copying Yarn packages to the ${YARN_CACHE_DIR} directory${COLOR_RESET}"
+    echo
+    set -e
+    set -x
+    cp -r ./node_modules $YARN_CACHE_DIR/
+    echo "${COLOR_BLUE}Yarn packages copied successfully${COLOR_RESET}"
+    set +x
     # Clean up
+    remove_npm_and_yarn
     rm -rf "${TEMP_AIRFLOW_DIR}"
+    set +e
+}
+
+function install_npm_and_yarn() {
+    echo
+    echo "${COLOR_BLUE}Installing npm and yarn${COLOR_RESET}"
+    echo
+    set -e
+    set -x
+    # Install npm
+    apt-get update
+    apt-get install -y npm
+    # Install yarn
+    npm install -g yarn
+    echo "${COLOR_BLUE}npm and yarn installed successfully${COLOR_RESET}"
+    set +x
+    set +e
+}
+
+function remove_npm_and_yarn() {
+    echo
+    echo "${COLOR_BLUE}Removing npm and yarn${COLOR_RESET}"
+    echo
+    set +e
+    set -x
+    # Remove yarn
+    npm cache clean --force
+    npm uninstall -g yarn
+    # Remove npm
+    apt-get remove -y npm
+    echo "${COLOR_BLUE}npm and yarn removed successfully${COLOR_RESET}"
+    set +x
+    set -e
 }
 
 common::get_colors
 
+install_npm_and_yarn
 install_yarn_dependencies_from_branch_tip
 EOF
 
@@ -1488,13 +1533,28 @@ ARG DOCKER_CONTEXT_FILES="Dockerfile"
 ARG AIRFLOW_HOME
 ARG AIRFLOW_USER_HOME_DIR
 ARG AIRFLOW_UID
+ARG AIRFLOW_REPO=apache/airflow
+ARG AIRFLOW_BRANCH=main
 
 RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \
        --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" && \
     mkdir -p ${AIRFLOW_HOME} && chown -R "airflow:0" "${AIRFLOW_USER_HOME_DIR}" ${AIRFLOW_HOME}
 
 COPY --chown=${AIRFLOW_UID}:0 ${DOCKER_CONTEXT_FILES} /docker-context-files
 
+# Yarn cache
+ARG AIRFLOW_PRE_CACHED_YARN_PACKAGES="true"
+ENV AIRFLOW_PRE_CACHED_YARN_PACKAGES=${AIRFLOW_PRE_CACHED_YARN_PACKAGES}
+COPY --from=scripts install_yarn_dependencies_from_branch_tip.sh /scripts/docker/
+
+ENV YARN_CACHE_DIR="${AIRFLOW_USER_HOME_DIR}/.yarn-cache"
+RUN mkdir -p "${YARN_CACHE_DIR}" && \
+    chown -R "airflow:0" "${YARN_CACHE_DIR}"
+# We are installing Yarn dependencies here to make sure they are cached in the layer
+RUN if [[ ${AIRFLOW_PRE_CACHED_YARN_PACKAGES} == "true" ]]; then \
+        bash /scripts/docker/install_yarn_dependencies_from_branch_tip.sh; \
+    fi
+
 USER airflow
 
 ARG AIRFLOW_REPO=apache/airflow
@@ -1513,8 +1573,6 @@ ARG PIP_PROGRESS_BAR
 # By default we do not use pre-cached packages, but in CI/Breeze environment we override this to speed up
 # builds in case pyproject.toml changed. This is pure optimisation of CI/Breeze builds.
 ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"
-# By default we do not use pre-cached yarn packages, but in CI/Breeze environment we override this to speed up
-ARG AIRFLOW_PRE_CACHED_YARN_PACKAGES="true"
 # This is airflow version that is put in the label of the image build
 ARG AIRFLOW_VERSION
 # By default latest released version of airflow is installed (when empty) but this value can be overridden
@@ -1553,7 +1611,6 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
     UV_HTTP_TIMEOUT=${UV_HTTP_TIMEOUT} \
     AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \
     AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \
-    AIRFLOW_PRE_CACHED_YARN_PACKAGES=${AIRFLOW_PRE_CACHED_YARN_PACKAGES} \
     AIRFLOW_VERSION=${AIRFLOW_VERSION} \
     AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} \
     AIRFLOW_VERSION_SPECIFICATION=${AIRFLOW_VERSION_SPECIFICATION} \
@@ -1579,8 +1636,7 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
 # Copy all scripts required for installation - changing any of those should lead to
 # rebuilding from here
 COPY --from=scripts common.sh install_packaging_tools.sh \
-     install_airflow_dependencies_from_branch_tip.sh create_prod_venv.sh \
-     install_yarn_dependencies_from_branch_tip.sh /scripts/docker/
+     install_airflow_dependencies_from_branch_tip.sh create_prod_venv.sh /scripts/docker/
 
 # We can set this value to true in case we want to install .whl/.tar.gz packages placed in the
 # docker-context-files folder. This can be done for both additional packages you want to install
@@ -1618,12 +1674,6 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \
         bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
     fi
 
-# We are installing Yarn dependencies here to make sure they are cached in the layer
-RUN if [[ ${AIRFLOW_PRE_CACHED_YARN_PACKAGES} == "true" ]]; then \
-        bash /scripts/docker/install_yarn_dependencies_from_branch_tip.sh; \
-    fi
-
-
 COPY --chown=airflow:0 ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO}
 
 # Add extra python dependencies

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -1186,22 +1186,67 @@ function install_yarn_dependencies_from_branch_tip() {
     local TEMP_AIRFLOW_DIR
     TEMP_AIRFLOW_DIR=$(mktemp -d)
     # Download the source code from the specified branch
-    set +e
+    set -e
     set -x
     curl -fsSL "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" | \
         tar xz -C "${TEMP_AIRFLOW_DIR}" --strip 1
     # Install Yarn dependencies
     cd "${TEMP_AIRFLOW_DIR}/airflow/www"
     yarn install --frozen-lockfile
     set +x
-    set -e
+    set +e
     echo "${COLOR_BLUE}Yarn dependencies installed successfully${COLOR_RESET}"
+
+    # Copy Yarn packages to the .yarn-cache directory
+    echo
+    echo "${COLOR_BLUE}Copying Yarn packages to the ${YARN_CACHE_DIR} directory${COLOR_RESET}"
+    echo
+    set -e
+    set -x
+    cp -r ./node_modules $YARN_CACHE_DIR/
+    echo "${COLOR_BLUE}Yarn packages copied successfully${COLOR_RESET}"
+    set +x
     # Clean up
+    remove_npm_and_yarn
     rm -rf "${TEMP_AIRFLOW_DIR}"
+    set +e
+}
+
+function install_npm_and_yarn() {
+    echo
+    echo "${COLOR_BLUE}Installing npm and yarn${COLOR_RESET}"
+    echo
+    set -e
+    set -x
+    # Install npm
+    apt-get update
+    apt-get install -y npm
+    # Install yarn
+    npm install -g yarn
+    echo "${COLOR_BLUE}npm and yarn installed successfully${COLOR_RESET}"
+    set +x
+    set +e
+}
+
+function remove_npm_and_yarn() {
+    echo
+    echo "${COLOR_BLUE}Removing npm and yarn${COLOR_RESET}"
+    echo
+    set +e
+    set -x
+    # Remove yarn
+    npm cache clean --force
+    npm uninstall -g yarn
+    # Remove npm
+    apt-get remove -y npm
+    echo "${COLOR_BLUE}npm and yarn removed successfully${COLOR_RESET}"
+    set +x
+    set -e
 }
 
 common::get_colors
 
+install_npm_and_yarn
 install_yarn_dependencies_from_branch_tip
 EOF
 
@@ -1311,7 +1356,6 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main"
 # It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH environment variable.
 ARG AIRFLOW_CI_BUILD_EPOCH="10"
 ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
-ARG AIRFLOW_PRE_CACHED_YARN_PACKAGES="true"
 # Setup PIP
 # By default PIP install run without cache to make image smaller
 ARG PIP_NO_CACHE_DIR="true"
@@ -1345,7 +1389,6 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
     DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \
     AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} \
     AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \
-    AIRFLOW_PRE_CACHED_YARN_PACKAGES=${AIRFLOW_PRE_CACHED_YARN_PACKAGES} \
     AIRFLOW_VERSION=${AIRFLOW_VERSION} \
     AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
     AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \
@@ -1387,11 +1430,20 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \
         bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
     fi
 
+# Yarn cache
+ARG AIRFLOW_PRE_CACHED_YARN_PACKAGES="true"
+ENV AIRFLOW_PRE_CACHED_YARN_PACKAGES=${AIRFLOW_PRE_CACHED_YARN_PACKAGES}
+COPY --from=scripts install_yarn_dependencies_from_branch_tip.sh /scripts/docker/
+
+ENV YARN_CACHE_DIR="${AIRFLOW_USER_HOME_DIR}/.yarn-cache"
+RUN mkdir -p "${YARN_CACHE_DIR}" && \
+    chown -R "airflow:0" "${YARN_CACHE_DIR}"
 # We are installing Yarn dependencies here to make sure they are cached in the layer
 RUN if [[ ${AIRFLOW_PRE_CACHED_YARN_PACKAGES} == "true" ]]; then \
         bash /scripts/docker/install_yarn_dependencies_from_branch_tip.sh; \
     fi
 
+
 # Here we fix the versions so all subsequent commands will use the versions
 # from the sources
 # You can swap comments between those two args to test pip from the main version

diff --git a/scripts/ci/pre_commit/compile_www_assets.py b/scripts/ci/pre_commit/compile_www_assets.py
@@ -56,6 +56,7 @@ def get_directory_hash(directory: Path, skip_path_regexp: str | None = None) ->
     www_directory = AIRFLOW_SOURCES_PATH / "airflow" / "www"
     node_modules_directory = www_directory / "node_modules"
     dist_directory = www_directory / "static" / "dist"
+    YARN_CACHE_DIR = os.getenv("YARN_CACHE_DIR", f"{Path.home()}/.yarn-cache")
     WWW_HASH_FILE.parent.mkdir(exist_ok=True, parents=True)
     if node_modules_directory.exists() and dist_directory.exists():
         old_hash = WWW_HASH_FILE.read_text() if WWW_HASH_FILE.exists() else ""
@@ -68,7 +69,12 @@ def get_directory_hash(directory: Path, skip_path_regexp: str | None = None) ->
         shutil.rmtree(dist_directory, ignore_errors=True)
     env = os.environ.copy()
     env["FORCE_COLOR"] = "true"
-    if os.getenv("AIRFLOW_PRE_CACHED_YARN_PACKAGES") != "true":
+    if os.getenv("AIRFLOW_PRE_CACHED_YARN_PACKAGES") == "true":
+        # Copy yarn-cache to node_modules from yarn-cache
+        shutil.copytree(YARN_CACHE_DIR, www_directory)
+        # Remove the yarn-cache directory to reduce the size of the docker image and prevent duplication
+        shutil.rmtree(YARN_CACHE_DIR, ignore_errors=True)
+    else:
         subprocess.check_call(["yarn", "install", "--frozen-lockfile"], cwd=os.fspath(www_directory))
     subprocess.check_call(["yarn", "run", "build"], cwd=os.fspath(www_directory), env=env)
     new_hash = get_directory_hash(www_directory, skip_path_regexp=r".*node_modules.*")

diff --git a/scripts/docker/install_yarn_dependencies_from_branch_tip.sh b/scripts/docker/install_yarn_dependencies_from_branch_tip.sh
@@ -37,20 +37,67 @@ function install_yarn_dependencies_from_branch_tip() {
     local TEMP_AIRFLOW_DIR
     TEMP_AIRFLOW_DIR=$(mktemp -d)
     # Download the source code from the specified branch
-    set +e
+    set -e
     set -x
     curl -fsSL "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" | \
         tar xz -C "${TEMP_AIRFLOW_DIR}" --strip 1
     # Install Yarn dependencies
     cd "${TEMP_AIRFLOW_DIR}/airflow/www"
     yarn install --frozen-lockfile
     set +x
-    set -e
+    set +e
     echo "${COLOR_BLUE}Yarn dependencies installed successfully${COLOR_RESET}"
+
+    # Copy Yarn packages to the .yarn-cache directory
+    echo
+    echo "${COLOR_BLUE}Copying Yarn packages to the ${YARN_CACHE_DIR} directory${COLOR_RESET}"
+    echo
+    set -e
+    set -x
+    cp -r ./node_modules $YARN_CACHE_DIR/
+    echo "${COLOR_BLUE}Yarn packages copied successfully${COLOR_RESET}"
+    set +x
     # Clean up
+    remove_npm_and_yarn
     rm -rf "${TEMP_AIRFLOW_DIR}"
+    set +e
+}
+
+# Install npm and yarn function
+function install_npm_and_yarn() {
+    echo
+    echo "${COLOR_BLUE}Installing npm and yarn${COLOR_RESET}"
+    echo
+    set -e
+    set -x
+    # Install npm
+    apt-get update
+    apt-get install -y npm
+    # Install yarn
+    npm install -g yarn
+    echo "${COLOR_BLUE}npm and yarn installed successfully${COLOR_RESET}"
+    set +x
+    set +e
+}
+
+# Remove yarn and npm function
+function remove_npm_and_yarn() {
+    echo
+    echo "${COLOR_BLUE}Removing npm and yarn${COLOR_RESET}"
+    echo
+    set +e
+    set -x
+    # Remove yarn
+    npm cache clean --force
+    npm uninstall -g yarn
+    # Remove npm
+    apt-get remove -y npm
+    echo "${COLOR_BLUE}npm and yarn removed successfully${COLOR_RESET}"
+    set +x
+    set -e
 }
 
 common::get_colors
 
+install_npm_and_yarn
 install_yarn_dependencies_from_branch_tip