Support DOCKER_HOST variable passing to Breeze commands (#36011)

In some cases you might want to override the default way how
Docker determines where to connect by seting DOCKER_HOST variable.

This is generally working out of the box by having the variable
set, but then it does not work well with `--builder` settings
in `builder` plugin. This PR adds support for DOCKER_HOST to be added
to all shell and build commands by disabling the builder when the
variable is set and explicit forwarding of the DOCKER_HOST
variable when running docker commands. You can also override the
DOCKER_HOST variable via `--docker-host` command line switch.

Build arguments in Breeze commands have been sorted as well as their
click options as the command option list grew large enough to be
messy if they are not alphabetically sorted.

We also add automated support for docker-in-docker case which we use in
some tests - the Docker-In-Docker should now work in most cases even
if you set DOCKER_HOST:

* if the ~/.docker/run/docker.sock is used - we change the mapping
  of socket to /var/run/docker.sock
* if DOCKER_HOST is not a "unix://" socket, we pass it through
  (hoping it will **just** work as tcp:// or ssh:// - named pipes
  are not forwardeable anyway)
* if Rootless Docker is used, we follow the rootless docker standard
  for XDG_RUNTIME_DIR/docker.sock (fallback to /run/(UID)/docker.sock

Without DOCKER_HOST we just forward /var/run/docker.sock which
should work also on MacOS - with, or without root-owned socket..

This gets a litte tricky because the way it works on MacOS /
DockerDesktop is a little magicalx. If you set DOCKER_HOST manually to
point to your HOME_DIR socket, you are actually supposed to forward
`/var/run/docker.sock` not the original socket because mappings you
define on MacOS are actually the VM mappings (and your home dir is
mounted to VM - EXCEPT the docker socket which is available on the VM as
/var/run/docker.sock)

More about it in this issue: docker/for-mac#6529
Also docs about MacOS Docker: https://docs.docker.com/desktop/mac/permission-requirements/

(cherry picked from commit e331559)

dev/breeze/src/airflow_breeze/commands/ci_image_commands.py

@@ -52,6 +52,7 @@
     option_dev_apt_command,
     option_dev_apt_deps,
     option_docker_cache,
+    option_docker_host,
     option_dry_run,
     option_eager_upgrade_additional_requirements,
     option_github_repository,
@@ -85,6 +86,7 @@
 from airflow_breeze.utils.docker_command_utils import (
     build_cache,
     check_remote_ghcr_io_commands,
+    get_docker_build_env,
     make_sure_builder_configured,
     perform_environment_checks,
     prepare_docker_build_command,
@@ -234,87 +236,86 @@ def get_exitcode(status: int) -> int:
 
 
 @ci_image.command(name="build")
-@option_python
-@option_debian_version
-@option_upgrade_to_newer_dependencies
-@option_upgrade_on_failure
-@option_platform_multiple
-@option_github_token
-@option_docker_cache
-@option_image_tag_for_building
-@option_prepare_buildx_cache
-@option_push
-@option_install_providers_from_sources
 @option_additional_airflow_extras
-@option_additional_dev_apt_deps
-@option_additional_python_deps
 @option_additional_dev_apt_command
+@option_additional_dev_apt_deps
 @option_additional_dev_apt_env
-@option_builder
+@option_additional_pip_install_flags
+@option_additional_python_deps
+@option_airflow_constraints_location
+@option_airflow_constraints_mode_ci
+@option_airflow_constraints_reference_build
+@option_answer
 @option_build_progress
+@option_build_timeout_minutes
+@option_builder
 @option_commit_sha
+@option_debian_version
+@option_debug_resources
 @option_dev_apt_command
 @option_dev_apt_deps
-@option_python_image
+@option_docker_cache
+@option_docker_host
+@option_dry_run
 @option_eager_upgrade_additional_requirements
-@option_airflow_constraints_location
-@option_airflow_constraints_mode_ci
-@option_airflow_constraints_reference_build
-@option_tag_as_latest
-@option_additional_pip_install_flags
 @option_github_repository
-@option_version_suffix_for_pypi_ci
-@option_build_timeout_minutes
-@option_run_in_parallel
-@option_parallelism
-@option_skip_cleanup
-@option_debug_resources
+@option_github_token
+@option_image_tag_for_building
 @option_include_success_outputs
+@option_install_providers_from_sources
+@option_parallelism
+@option_platform_multiple
+@option_prepare_buildx_cache
+@option_push
+@option_python
+@option_python_image
 @option_python_versions
+@option_run_in_parallel
+@option_skip_cleanup
+@option_tag_as_latest
+@option_upgrade_on_failure
+@option_upgrade_to_newer_dependencies
 @option_verbose
-@option_dry_run
-@option_answer
+@option_version_suffix_for_pypi_ci
 def build(
-    # Build options
-    python: str,
-    debian_version: str,
-    upgrade_to_newer_dependencies: bool,
-    upgrade_on_failure: bool,
-    platform: str | None,
-    github_token: str | None,
-    docker_cache: str,
-    image_tag: str,
-    prepare_buildx_cache: bool,
-    push: bool,
-    install_providers_from_sources: bool,
     additional_airflow_extras: str | None,
-    additional_dev_apt_deps: str | None,
-    additional_python_deps: str | None,
     additional_dev_apt_command: str | None,
+    additional_dev_apt_deps: str | None,
     additional_dev_apt_env: str | None,
-    builder: str,
+    additional_pip_install_flags: str | None,
+    additional_python_deps: str | None,
+    airflow_constraints_location: str | None,
+    airflow_constraints_mode: str,
+    airflow_constraints_reference: str,
     build_progress: str,
+    build_timeout_minutes: int | None,
+    builder: str,
     commit_sha: str | None,
+    debian_version: str,
+    debug_resources: bool,
     dev_apt_command: str | None,
     dev_apt_deps: str | None,
+    docker_cache: str,
+    docker_host: str | None,
     eager_upgrade_additional_requirements: str | None,
-    airflow_constraints_location: str | None,
-    airflow_constraints_mode: str,
-    airflow_constraints_reference: str,
-    tag_as_latest: bool,
-    additional_pip_install_flags: str | None,
     github_repository: str,
+    github_token: str | None,
+    image_tag: str,
+    include_success_outputs,
+    install_providers_from_sources: bool,
+    parallelism: int,
+    platform: str | None,
+    prepare_buildx_cache: bool,
+    push: bool,
+    python: str,
     python_image: str | None,
-    version_suffix_for_pypi: str,
-    # Parallel building
+    python_versions: str,
     run_in_parallel: bool,
-    parallelism: int,
     skip_cleanup: bool,
-    debug_resources: bool,
-    include_success_outputs,
-    python_versions: str,
-    # Other options
-    build_timeout_minutes: int | None,
+    tag_as_latest: bool,
+    upgrade_on_failure: bool,
+    upgrade_to_newer_dependencies: bool,
+    version_suffix_for_pypi: str,
 ):
     """Build CI image. Include building multiple images for all python versions."""
 
@@ -351,34 +352,35 @@ def run_build(ci_image_params: BuildCiParams) -> None:
     check_remote_ghcr_io_commands()
     fix_group_permissions()
     base_build_params = BuildCiParams(
-        force_build=True,
-        python=python,
-        debian_version=debian_version,
-        upgrade_to_newer_dependencies=upgrade_to_newer_dependencies,
-        upgrade_on_failure=upgrade_on_failure,
-        github_token=github_token,
-        docker_cache=docker_cache,
-        image_tag=image_tag,
-        prepare_buildx_cache=prepare_buildx_cache,
-        push=push,
-        install_providers_from_sources=install_providers_from_sources,
         additional_airflow_extras=additional_airflow_extras,
-        additional_python_deps=additional_python_deps,
         additional_dev_apt_command=additional_dev_apt_command,
         additional_dev_apt_env=additional_dev_apt_env,
-        builder=builder,
+        additional_pip_install_flags=additional_pip_install_flags,
+        additional_python_deps=additional_python_deps,
+        airflow_constraints_location=airflow_constraints_location,
+        airflow_constraints_mode=airflow_constraints_mode,
+        airflow_constraints_reference=airflow_constraints_reference,
         build_progress=build_progress,
+        builder=builder,
         commit_sha=commit_sha,
+        debian_version=debian_version,
         dev_apt_command=dev_apt_command,
         dev_apt_deps=dev_apt_deps,
+        docker_cache=docker_cache,
+        docker_host=docker_host,
         eager_upgrade_additional_requirements=eager_upgrade_additional_requirements,
-        airflow_constraints_location=airflow_constraints_location,
-        airflow_constraints_mode=airflow_constraints_mode,
-        airflow_constraints_reference=airflow_constraints_reference,
-        tag_as_latest=tag_as_latest,
-        additional_pip_install_flags=additional_pip_install_flags,
+        force_build=True,
         github_repository=github_repository,
+        github_token=github_token,
+        image_tag=image_tag,
+        install_providers_from_sources=install_providers_from_sources,
+        prepare_buildx_cache=prepare_buildx_cache,
+        push=push,
+        python=python,
         python_image=python_image,
+        tag_as_latest=tag_as_latest,
+        upgrade_on_failure=upgrade_on_failure,
+        upgrade_to_newer_dependencies=upgrade_to_newer_dependencies,
         version_suffix_for_pypi=version_suffix_for_pypi,
     )
     if platform:
@@ -727,8 +729,7 @@ def run_build_ci_image(
             output=output,
         )
     else:
-        env = os.environ.copy()
-        env["DOCKER_BUILDKIT"] = "1"
+        env = get_docker_build_env(ci_image_params)
         subprocess.run(
             [
                 sys.executable,
@@ -799,15 +800,16 @@ def rebuild_or_pull_ci_image_if_needed(command_params: ShellParams | BuildCiPara
         BUILD_CACHE_DIR, command_params.airflow_branch, f".built_{command_params.python}"
     )
     ci_image_params = BuildCiParams(
-        python=command_params.python,
         builder=command_params.builder,
+        docker_host=command_params.docker_host,
+        force_build=command_params.force_build,
         github_repository=command_params.github_repository,
-        upgrade_to_newer_dependencies=False,
         image_tag=command_params.image_tag,
         platform=command_params.platform,
-        force_build=command_params.force_build,
-        skip_provider_dependencies_check=command_params.skip_provider_dependencies_check,
+        python=command_params.python,
         skip_image_upgrade_check=command_params.skip_image_upgrade_check,
+        skip_provider_dependencies_check=command_params.skip_provider_dependencies_check,
+        upgrade_to_newer_dependencies=False,
         warn_image_upgrade_needed=command_params.warn_image_upgrade_needed,
     )
     if command_params.image_tag is not None and command_params.image_tag != "latest":

dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py

@@ -37,6 +37,7 @@
                 "--docker-cache",
                 "--version-suffix-for-pypi",
                 "--build-progress",
+                "--docker-host",
             ],
         },
         {

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -53,6 +53,7 @@
     option_celery_flower,
     option_database_isolation,
     option_db_reset,
+    option_docker_host,
     option_downgrade_sqlalchemy,
     option_dry_run,
     option_executor_shell,
@@ -180,6 +181,7 @@ def run(self):
 @option_celery_flower
 @option_database_isolation
 @option_db_reset
+@option_docker_host
 @option_downgrade_sqlalchemy
 @option_dry_run
 @option_executor_shell
@@ -220,6 +222,7 @@ def shell(
     database_isolation: bool,
     db_reset: bool,
     downgrade_sqlalchemy: bool,
+    docker_host: str | None,
     executor: str,
     extra_args: tuple,
     force_build: bool,
@@ -272,6 +275,7 @@ def shell(
         database_isolation=database_isolation,
         db_reset=db_reset,
         downgrade_sqlalchemy=downgrade_sqlalchemy,
+        docker_host=docker_host,
         executor=executor,
         extra_args=extra_args if not max_time else ["exit"],
         force_build=force_build,
@@ -330,6 +334,7 @@ def shell(
 @option_celery_flower
 @option_database_isolation
 @option_db_reset
+@option_docker_host
 @option_dry_run
 @option_executor_start_airflow
 @option_force_build
@@ -362,6 +367,7 @@ def start_airflow(
     database_isolation: bool,
     db_reset: bool,
     dev_mode: bool,
+    docker_host: str | None,
     executor: str,
     extra_args: tuple,
     force_build: bool,
@@ -410,6 +416,7 @@ def start_airflow(
         database_isolation=database_isolation,
         db_reset=db_reset,
         dev_mode=dev_mode,
+        docker_host=docker_host,
         executor=executor,
         extra_args=extra_args,
         force_build=force_build,

dev/breeze/src/airflow_breeze/commands/developer_commands_config.py

@@ -45,6 +45,7 @@
             "options": [
                 "--project-name",
                 "--restart",
+                "--docker-host",
             ],
         },
         {
@@ -87,6 +88,7 @@
             "options": [
                 "--project-name",
                 "--restart",
+                "--docker-host",
             ],
         },
         {
@@ -195,6 +197,7 @@
             "options": [
                 "--project-name",
                 "--restart",
+                "--docker-host",
             ],
         },
         {

dev/breeze/src/airflow_breeze/commands/main_command.py

@@ -33,6 +33,7 @@
     option_builder,
     option_database_isolation,
     option_db_reset,
+    option_docker_host,
     option_dry_run,
     option_forward_credentials,
     option_github_repository,
@@ -108,6 +109,7 @@ def get_command(self, ctx: Context, cmd_name: str):
 @option_builder
 @option_database_isolation
 @option_db_reset
+@option_docker_host
 @option_dry_run
 @option_forward_credentials
 @option_github_repository