Remove pre-commit from devel extra of Airflow (apache#44059)

Having pre-commit as devel dependency is not a good idea because it might conflict with the one installed globall when you install a local venv. Instead of installing it as a dependency, we install it separately in CI and CI image - together with UV and pre-comit-uv to make it fast and keep consistency and keep it in the same version in the development environment. We still have TODO to keep it synchronized and updated automatically in all places and in github action - that will be done in a separate PR.
LefterisXefteris · Jan 5, 2025 · b49c8e9 · b49c8e9
1 parent a70742e
commit b49c8e9
Show file tree

Hide file tree

Showing 11 changed files with 119 additions and 59 deletions.
diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml
@@ -24,7 +24,7 @@ inputs:
     default: 3.9
   uv-version:
     description: 'uv version to use'
-    default: 0.4.30
+    default: 0.5.2
   pre-commit-version:
     description: 'pre-commit version to use'
     default: 4.0.1
@@ -36,11 +36,10 @@ runs:
   steps:
     - name: Install pre-commit, uv, and pre-commit-uv
       shell: bash
-      run: >
-        pip install
-        pre-commit==${{inputs.pre-commit-version}}
-        uv==${{inputs.uv-version}}
-        pre-commit-uv==${{inputs.pre-commit-uv-version}}
+      run: |
+        pip install uv==${{inputs.uv-version}} || true
+        uv tool install pre-commit==${{inputs.pre-commit-version}} --with uv==${{inputs.uv-version}} \
+        --with pre-commit-uv==${{inputs.pre-commit-uv-version}}
     - name: Cache pre-commit envs
       uses: actions/cache@v4
       with:

diff --git a/Dockerfile b/Dockerfile
@@ -55,7 +55,7 @@ ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm"
 # Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
 ARG AIRFLOW_PIP_VERSION=24.3.1
 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
-ARG AIRFLOW_UV_VERSION=0.5.1
+ARG AIRFLOW_UV_VERSION=0.5.2
 ARG AIRFLOW_USE_UV="false"
 ARG UV_HTTP_TIMEOUT="300"
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
@@ -606,6 +606,7 @@ function common::show_packaging_tool_version_and_location() {
 }
 
 function common::install_packaging_tools() {
+    : "${AIRFLOW_USE_UV:?Should be set}"
     if [[ "${VIRTUAL_ENV=}" != "" ]]; then
         echo
         echo "${COLOR_BLUE}Checking packaging tools in venv: ${VIRTUAL_ENV}${COLOR_RESET}"
@@ -658,8 +659,23 @@ function common::install_packaging_tools() {
             pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}"
         fi
     fi
-    # make sure that the venv/user in .local exists
-    mkdir -p "${HOME}/.local/bin"
+    if  [[ ${AIRFLOW_PRE_COMMIT_VERSION=} == "" ]]; then
+        echo
+        echo "${COLOR_BLUE}Installing latest pre-commit with pre-commit-uv uv${COLOR_RESET}"
+        echo
+        uv tool install pre-commit --with pre-commit-uv --with uv
+        # make sure that the venv/user in .local exists
+        mkdir -p "${HOME}/.local/bin"
+    else
+        echo
+        echo "${COLOR_BLUE}Installing predefined versions of pre-commit with pre-commit-uv and uv:${COLOR_RESET}"
+        echo "${COLOR_BLUE}pre_commit(${AIRFLOW_PRE_COMMIT_VERSION}) uv(${AIRFLOW_UV_VERSION}) pre_commit-uv(${AIRFLOW_PRE_COMMIT_UV_VERSION})${COLOR_RESET}"
+        echo
+        uv tool install "pre-commit==${AIRFLOW_PRE_COMMIT_VERSION}" \
+            --with "uv==${AIRFLOW_UV_VERSION}" --with "pre-commit-uv==${AIRFLOW_PRE_COMMIT_UV_VERSION}"
+        # make sure that the venv/user in .local exists
+        mkdir -p "${HOME}/.local/bin"
+    fi
 }
 
 function common::import_trusted_gpg() {

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -547,6 +547,7 @@ function common::show_packaging_tool_version_and_location() {
 }
 
 function common::install_packaging_tools() {
+    : "${AIRFLOW_USE_UV:?Should be set}"
     if [[ "${VIRTUAL_ENV=}" != "" ]]; then
         echo
         echo "${COLOR_BLUE}Checking packaging tools in venv: ${VIRTUAL_ENV}${COLOR_RESET}"
@@ -599,8 +600,23 @@ function common::install_packaging_tools() {
             pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}"
         fi
     fi
-    # make sure that the venv/user in .local exists
-    mkdir -p "${HOME}/.local/bin"
+    if  [[ ${AIRFLOW_PRE_COMMIT_VERSION=} == "" ]]; then
+        echo
+        echo "${COLOR_BLUE}Installing latest pre-commit with pre-commit-uv uv${COLOR_RESET}"
+        echo
+        uv tool install pre-commit --with pre-commit-uv --with uv
+        # make sure that the venv/user in .local exists
+        mkdir -p "${HOME}/.local/bin"
+    else
+        echo
+        echo "${COLOR_BLUE}Installing predefined versions of pre-commit with pre-commit-uv and uv:${COLOR_RESET}"
+        echo "${COLOR_BLUE}pre_commit(${AIRFLOW_PRE_COMMIT_VERSION}) uv(${AIRFLOW_UV_VERSION}) pre_commit-uv(${AIRFLOW_PRE_COMMIT_UV_VERSION})${COLOR_RESET}"
+        echo
+        uv tool install "pre-commit==${AIRFLOW_PRE_COMMIT_VERSION}" \
+            --with "uv==${AIRFLOW_UV_VERSION}" --with "pre-commit-uv==${AIRFLOW_PRE_COMMIT_UV_VERSION}"
+        # make sure that the venv/user in .local exists
+        mkdir -p "${HOME}/.local/bin"
+    fi
 }
 
 function common::import_trusted_gpg() {
@@ -1338,10 +1354,14 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \
 # Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
 ARG AIRFLOW_PIP_VERSION=24.3.1
 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
-ARG AIRFLOW_UV_VERSION=0.5.1
+ARG AIRFLOW_UV_VERSION=0.5.2
+# TODO(potiuk): automate with upgrade check (possibly)
+ARG AIRFLOW_PRE_COMMIT_VERSION="4.0.1"
+ARG AIRFLOW_PRE_COMMIT_UV_VERSION="4.1.4"
 
 ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
-    AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION}
+    AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \
+    AIRFLOW_PRE_COMMIT_VERSION=${AIRFLOW_PRE_COMMIT_VERSION}
 
 # The PATH is needed for PIPX to find the tools installed
 ENV PATH="/root/.local/bin:${PATH}"

diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md
@@ -448,7 +448,7 @@ can be used for CI images:
 | `ADDITIONAL_DEV_APT_DEPS`         |                            | Additional apt dev dependencies installed in the first part of the image                                                                                   |
 | `ADDITIONAL_DEV_APT_ENV`          |                            | Additional env variables defined when installing dev deps                                                                                                  |
 | `AIRFLOW_PIP_VERSION`             | `24.3.1`                   | PIP version used.                                                                                                                                          |
-| `AIRFLOW_UV_VERSION`              | `0.5.1`                    | UV version used.                                                                                                                                           |
+| `AIRFLOW_UV_VERSION`              | `0.5.2`                    | UV version used.                                                                                                                                           |
 | `AIRFLOW_USE_UV`                  | `true`                     | Whether to use UV for installation.                                                                                                                        |
 | `PIP_PROGRESS_BAR`                | `on`                       | Progress bar for PIP installation                                                                                                                          |
 

diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -230,7 +230,7 @@ class VersionedFile(NamedTuple):
 
 
 AIRFLOW_PIP_VERSION = "24.3.1"
-AIRFLOW_UV_VERSION = "0.5.1"
+AIRFLOW_UV_VERSION = "0.5.2"
 AIRFLOW_USE_UV = False
 # TODO: automate these as well
 WHEEL_VERSION = "0.44.0"

diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -208,7 +208,7 @@
 ALLOWED_INSTALL_MYSQL_CLIENT_TYPES = ["mariadb", "mysql"]
 
 PIP_VERSION = "24.3.1"
-UV_VERSION = "0.5.1"
+UV_VERSION = "0.5.2"
 
 DEFAULT_UV_HTTP_TIMEOUT = 300
 DEFAULT_WSL2_HTTP_TIMEOUT = 900

diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py
@@ -218,48 +218,58 @@ def assert_pre_commit_installed():
 
     python_executable = sys.executable
     get_console().print(f"[info]Checking pre-commit installed for {python_executable}[/]")
-    command_result = run_command(
-        ["pre-commit", "--version"],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    if command_result.returncode == 0:
-        if command_result.stdout:
-            pre_commit_version = command_result.stdout.split(" ")[1].strip()
-            if Version(pre_commit_version) >= Version(min_pre_commit_version):
-                get_console().print(
-                    f"\n[success]Package pre_commit is installed. "
-                    f"Good version {pre_commit_version} (>= {min_pre_commit_version})[/]\n"
-                )
+    need_to_reinstall_precommit = False
+    try:
+        command_result = run_command(
+            ["pre-commit", "--version"],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        if command_result.returncode == 0:
+            if command_result.stdout:
+                pre_commit_version = command_result.stdout.split(" ")[1].strip()
+                if Version(pre_commit_version) >= Version(min_pre_commit_version):
+                    get_console().print(
+                        f"\n[success]Package pre_commit is installed. "
+                        f"Good version {pre_commit_version} (>= {min_pre_commit_version})[/]\n"
+                    )
+                else:
+                    get_console().print(
+                        f"\n[error]Package name pre_commit version is wrong. It should be"
+                        f"aat least {min_pre_commit_version} and is {pre_commit_version}.[/]\n\n"
+                    )
+                    sys.exit(1)
+                if "pre-commit-uv" not in command_result.stdout:
+                    get_console().print(
+                        "\n[warning]You can significantly improve speed of installing your pre-commit envs "
+                        "by installing `pre-commit-uv` with it.[/]\n"
+                    )
+                    get_console().print(
+                        "\n[warning]With uv you can install it with:[/]\n\n"
+                        "        uv tool install pre-commit --with pre-commit-uv --force-reinstall\n"
+                    )
+                    get_console().print(
+                        "\n[warning]With pipx you can install it with:[/]\n\n"
+                        "        pipx inject pre-commit pre-commit-uv # optionally if you want to use uv to "
+                        "install virtualenvs\n"
+                    )
             else:
                 get_console().print(
-                    f"\n[error]Package name pre_commit version is wrong. It should be"
-                    f"aat least {min_pre_commit_version} and is {pre_commit_version}.[/]\n\n"
-                )
-                sys.exit(1)
-            if "pre-commit-uv" not in command_result.stdout:
-                get_console().print(
-                    "\n[warning]You can significantly improve speed of installing your pre-commit envs "
-                    "by installing `pre-commit-uv` with it.[/]\n"
-                )
-                get_console().print(
-                    "\n[warning]With uv you can install it with:[/]\n\n"
-                    "        uv tool install pre-commit --with pre-commit-uv --force-reinstall\n"
-                )
-                get_console().print(
-                    "\n[warning]With pipx you can install it with:[/]\n\n"
-                    "        pipx inject pre-commit pre-commit-uv # optionally if you want to use uv to "
-                    "install virtualenvs\n"
+                    "\n[warning]Could not determine version of pre-commit. You might need to update it![/]\n"
                 )
         else:
-            get_console().print(
-                "\n[warning]Could not determine version of pre-commit. You might need to update it![/]\n"
-            )
-    else:
-        get_console().print("\n[error]Error checking for pre-commit-installation:[/]\n")
-        get_console().print(command_result.stderr)
-        get_console().print("\nMake sure to run:\n      breeze setup self-upgrade\n\n")
+            need_to_reinstall_precommit = True
+            get_console().print("\n[error]Error checking for pre-commit-installation:[/]\n")
+            get_console().print(command_result.stderr)
+    except FileNotFoundError as e:
+        need_to_reinstall_precommit = True
+        get_console().print(f"\n[error]Error checking for pre-commit-installation: [/]\n{e}\n")
+    if need_to_reinstall_precommit:
+        get_console().print("\n[info]Make sure to install pre-commit. For example by running\n\n")
+        get_console().print("uv tool install pre-commit\n")
+        get_console().print("Or if you prefer pipx:\n")
+        get_console().print("pipx install pre-commit")
         sys.exit(1)
 
 

diff --git a/hatch_build.py b/hatch_build.py
@@ -248,7 +248,6 @@
     ],
     "devel-static-checks": [
         "black>=23.12.0",
-        "pre-commit>=3.5.0",
         "ruff==0.7.3",
         "yamllint>=1.33.0",
     ],

diff --git a/scripts/ci/install_breeze.sh b/scripts/ci/install_breeze.sh
@@ -22,7 +22,7 @@ cd "$( dirname "${BASH_SOURCE[0]}" )/../../"
 PYTHON_ARG=""
 
 PIP_VERSION="24.3.1"
-UV_VERSION="0.5.1"
+UV_VERSION="0.5.2"
 if [[ ${PYTHON_VERSION=} != "" ]]; then
     PYTHON_ARG="--python=$(which python"${PYTHON_VERSION}") "
 fi

diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
@@ -118,6 +118,7 @@ function common::show_packaging_tool_version_and_location() {
 }
 
 function common::install_packaging_tools() {
+    : "${AIRFLOW_USE_UV:?Should be set}"
     if [[ "${VIRTUAL_ENV=}" != "" ]]; then
         echo
         echo "${COLOR_BLUE}Checking packaging tools in venv: ${VIRTUAL_ENV}${COLOR_RESET}"
@@ -170,8 +171,23 @@ function common::install_packaging_tools() {
             pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}"
         fi
     fi
-    # make sure that the venv/user in .local exists
-    mkdir -p "${HOME}/.local/bin"
+    if  [[ ${AIRFLOW_PRE_COMMIT_VERSION=} == "" ]]; then
+        echo
+        echo "${COLOR_BLUE}Installing latest pre-commit with pre-commit-uv uv${COLOR_RESET}"
+        echo
+        uv tool install pre-commit --with pre-commit-uv --with uv
+        # make sure that the venv/user in .local exists
+        mkdir -p "${HOME}/.local/bin"
+    else
+        echo
+        echo "${COLOR_BLUE}Installing predefined versions of pre-commit with pre-commit-uv and uv:${COLOR_RESET}"
+        echo "${COLOR_BLUE}pre_commit(${AIRFLOW_PRE_COMMIT_VERSION}) uv(${AIRFLOW_UV_VERSION}) pre_commit-uv(${AIRFLOW_PRE_COMMIT_UV_VERSION})${COLOR_RESET}"
+        echo
+        uv tool install "pre-commit==${AIRFLOW_PRE_COMMIT_VERSION}" \
+            --with "uv==${AIRFLOW_UV_VERSION}" --with "pre-commit-uv==${AIRFLOW_PRE_COMMIT_UV_VERSION}"
+        # make sure that the venv/user in .local exists
+        mkdir -p "${HOME}/.local/bin"
+    fi
 }
 
 function common::import_trusted_gpg() {

diff --git a/scripts/tools/setup_breeze b/scripts/tools/setup_breeze
@@ -27,7 +27,7 @@ COLOR_YELLOW=$'\e[33m'
 COLOR_BLUE=$'\e[34m'
 COLOR_RESET=$'\e[0m'
 
-UV_VERSION="0.5.1"
+UV_VERSION="0.5.2"
 
 function manual_instructions() {
     echo