Allow to override concrete --postgres-version/--mysql-version

.github/workflows/ci.yml

@@ -1070,14 +1070,14 @@ jobs:
   tests-postgres:
     timeout-minutes: 130
     name: >
-      DB:Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
+      DB:Postgres${{matrix.backend-version}},Py${{matrix.python-version}}:
       ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
     runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
     needs: [build-info, wait-for-ci-images]
     strategy:
       matrix:
         python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
-        postgres-version: "${{fromJson(needs.build-info.outputs.postgres-versions)}}"
+        backend-version: "${{fromJson(needs.build-info.outputs.postgres-versions)}}"
         exclude: "${{fromJson(needs.build-info.outputs.postgres-exclude)}}"
       fail-fast: false
     env:
@@ -1088,9 +1088,8 @@ jobs:
       DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
       BACKEND: "postgres"
       PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
-      POSTGRES_VERSION: "${{matrix.postgres-version}}"
-      BACKEND_VERSION: "${{matrix.postgres-version}}"
-      JOB_ID: "postgres-${{matrix.postgres-version}}-${{matrix.python-version}}"
+      BACKEND_VERSION: "${{matrix.backend-version}}"
+      JOB_ID: "postgres-${{matrix.backend-version}}-${{matrix.python-version}}"
       ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
     if: needs.build-info.outputs.run-tests == 'true'
     steps:
@@ -1116,7 +1115,7 @@ jobs:
           --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
       - name: "Tests ARM Pytest collection: ${{matrix.python-version}}"
         run: breeze testing db-tests --collect-only --remove-arm-packages
-        if: matrix.postgres-version == needs.build-info.outputs.default-postgres-version
+        if: matrix.backend-version == needs.build-info.outputs.default-postgres-version
       - name: >
           Post Tests success: Postgres"
         uses: ./.github/actions/post_tests_success
@@ -1144,7 +1143,6 @@ jobs:
       ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
       PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
-      POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
       BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
       DOWNGRADE_SQLALCHEMY: "true"
       JOB_ID: >
@@ -1195,7 +1193,6 @@ jobs:
       ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
       PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
-      POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
       BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
       UPGRADE_BOTO: "true"
       JOB_ID: >
@@ -1249,7 +1246,6 @@ jobs:
       ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
       PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
-      POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
       BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
       PYDANTIC: ${{ matrix.pydantic }}
       JOB_ID: >
@@ -1300,7 +1296,6 @@ jobs:
       ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
       PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
-      POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
       BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
       DOWNGRADE_PENDULUM: "true"
       JOB_ID: >
@@ -1353,7 +1348,6 @@ jobs:
       ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
       PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
-      POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
       BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
       AIRFLOW_ENABLE_AIP_44: "false"
       JOB_ID: >
@@ -1390,14 +1384,14 @@ jobs:
   tests-mysql:
     timeout-minutes: 130
     name: >
-      DB:MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}:
+      DB:MySQL${{matrix.backend-version}}, Py${{matrix.python-version}}:
       ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
     runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
     needs: [build-info, wait-for-ci-images]
     strategy:
       matrix:
         python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
-        mysql-version: "${{fromJson(needs.build-info.outputs.mysql-versions)}}"
+        backend-version: "${{fromJson(needs.build-info.outputs.mysql-versions)}}"
         exclude: "${{fromJson(needs.build-info.outputs.mysql-exclude)}}"
       fail-fast: false
     env:
@@ -1409,9 +1403,8 @@ jobs:
       BACKEND: "mysql"
       ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
-      MYSQL_VERSION: "${{matrix.mysql-version}}"
-      BACKEND_VERSION: "${{matrix.mysql-version}}"
-      JOB_ID: "mysql-${{matrix.mysql-version}}-${{matrix.python-version}}"
+      BACKEND_VERSION: "${{matrix.backend-version}}"
+      JOB_ID: "mysql-${{matrix.backend-version}}-${{matrix.python-version}}"
     if: needs.build-info.outputs.run-tests == 'true'
     steps:
       - name: "Cleanup repo"
@@ -1511,7 +1504,6 @@ jobs:
       BACKEND: "postgres"
       ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
-      POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
       BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
       JOB_ID: "integration-postgres"
       SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}"
@@ -1588,7 +1580,6 @@ jobs:
       BACKEND: "mysql"
       ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
-      MYSQL_VERSION: "${{needs.build-info.outputs.default-mysql-version}}"
       BACKEND_VERSION: "${{needs.build-info.outputs.default-mysql-version}}"
       JOB_ID: "integration-mysql"
       SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}"
@@ -1656,14 +1647,12 @@ jobs:
         env:
           BACKEND: "postgres"
           BACKEND_VERSION: ${{needs.build-info.outputs.default-postgres-version}}
-          POSTGRES_VERSION: ${{needs.build-info.outputs.default-postgres-version}}
       - name: >
           Tests: mysql:${{needs.build-info.outputs.default-python-version}}:Quarantined
         run: breeze testing tests || true
         env:
           BACKEND: "mysql"
           BACKEND_VERSION: ${{needs.build-info.outputs.default-mysql-version}}
-          MYSQL_VERSION: ${{needs.build-info.outputs.default-mysql-version}}
       - name: >
           Tests: sqlite:${{needs.build-info.outputs.default-python-version}}:Quarantined
         run: breeze testing tests || true

Dockerfile.ci

@@ -855,19 +855,19 @@ function environment_initialization() {
     if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} == "true" ]]; then
         return
     fi
-    if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
-        if [[ ${BACKEND:=} == "mssql" ]]; then
-            echo "${COLOR_RED}ARM platform is not supported for ${BACKEND} backend. Exiting.${COLOR_RESET}"
-            exit 1
-        fi
-    fi
-
     echo
     echo "${COLOR_BLUE}Running Initialization. Your basic configuration is:${COLOR_RESET}"
     echo
     echo "  * ${COLOR_BLUE}Airflow home:${COLOR_RESET} ${AIRFLOW_HOME}"
     echo "  * ${COLOR_BLUE}Airflow sources:${COLOR_RESET} ${AIRFLOW_SOURCES}"
-    echo "  * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}"
+    echo "  * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:=}"
+    if [[ ${BACKEND=} == "postgres" ]]; then
+        echo "  * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Postgres: ${POSTGRES_VERSION}"
+    elif [[ ${BACKEND=} == "mysql" ]]; then
+        echo "  * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} MySQL: ${MYSQL_VERSION}"
+    elif [[ ${BACKEND=} == "sqlite" ]]; then
+        echo "  * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Sqlite"
+    fi
     echo
 
     if [[ ${STANDALONE_DAG_PROCESSOR=} == "true" ]]; then

dev/breeze/doc/03_developer_tasks.rst

@@ -32,7 +32,7 @@ development environment (inside the Breeze container).
 
 You can use additional ``breeze`` flags to choose your environment. You can specify a Python
 version to use, and backend (the meta-data database). Thanks to that, with Breeze, you can recreate the same
-environments as we have in matrix builds in the CI.
+environments as we have in matrix builds in the CI. See next chapter for backend selection.
 
 For example, you can choose to run Python 3.8 tests with MySQL as backend and with mysql version 8
 as follows:
@@ -70,6 +70,40 @@ that user used last time.
 You can see which value of the parameters that can be stored persistently in cache marked with >VALUE<
 in the help of the commands (for example in output of ``breeze config --help``).
 
+Selecting Backend
+-----------------
+
+When you run breeze commands, you can additionally select which backend you want to use. Currently Airflow
+supports Sqlite, MySQL and Postgres as backends - MySQL and Postgres are supported in various versions.
+
+You can choose which backend to use by adding ``--backend`` flag and additionally you can select version
+of the backend, if you want to start a different version of backend (for example for ``--backend postgres``
+you can specify ``--postgres-version 13`` to start Postgres 13). The ``--help`` command in breeze commands
+will show you which backends are supported and which versions are available for each backend.
+
+The choice you made for backend and version are ``sticky`` - the last used selection is cached in the
+``.build`` folder and next time you run any of the ``breeze`` commands that use backend the will use the
+last selected backend and version.
+
+.. note::
+
+  You can also (temporarily for the time of running a single command) override the backend version
+  used via ``BACKEND_VERSION`` environment variable. This is used mostly in CI where we have common way of
+  running tests for all backends and we want to specify different parameters. In order to override the
+  backend version, it has to be a valid version for the backend you are using. For example if you set
+  ``BACKEND_VERSION`` to ``13`` and you are using ``--backend postgres``, Postgres 13 will be used, but
+  if you set ``BACKEND_VERSION`` to ``8.0`` and you are using ``--backend postgres``, the last used Postgres
+  version will be used.
+
+Breeze will inform you at startup which backend and version it is using:
+
+.. raw:: html
+
+    <div align="center">
+        <img src="images/version_information.png" width="640" alt="Version information printed by Breeze">
+    </div>
+
+
 Port Forwarding
 ---------------
 

dev/breeze/doc/images/output_setup_config.txt

@@ -1 +1 @@
-f59e1732fcaa9b00e19f763e2a25ae0f
+7400d6859de2b42523a65ed2b59a4883

dev/breeze/doc/images/output_shell.txt

@@ -1 +1 @@
-00ef3916470a22430b81c6b581ed9e78
+e2155a551cbd0ac33b7e1f3ddfed34f4

dev/breeze/doc/images/output_start-airflow.txt

@@ -1 +1 @@
-245c8a918e2e41e43dcf9f0be0eafcce
+bdc07b607fe352a291f387be297e78c7

dev/breeze/doc/images/output_testing_db-tests.txt

@@ -1 +1 @@
-1e2e23d641ddd95ea37ae86555cfe73b
+0a5e359c1162eef2d2031801d7f8ca5d

dev/breeze/doc/images/output_testing_integration-tests.txt

@@ -1 +1 @@
-02ba54f8b3ba2213343ea93f56e68585
+a15ab29d27810e93c440aee3972f83cc

dev/breeze/doc/images/output_testing_tests.txt

@@ -1 +1 @@
-3a60a3cd3ebc469f2fd2890450a6a936
+ad4de130e1b1e608a5ef26f89ddfd969

dev/breeze/src/airflow_breeze/commands/common_options.py

@@ -37,11 +37,12 @@
 )
 from airflow_breeze.utils.custom_param_types import (
     AnswerChoice,
+    BackendVersionChoice,
     BetterChoice,
     CacheableChoice,
     CacheableDefault,
     DryRunOption,
-    MySQLBackendVersionType,
+    MySQLBackendVersionChoice,
     NotVerifiedBetterChoice,
     UseAirflowVersionType,
     VerboseOption,
@@ -245,7 +246,7 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option,
     "-M",
     "--mysql-version",
     help="Version of MySQL used.",
-    type=MySQLBackendVersionType(ALLOWED_MYSQL_VERSIONS),
+    type=MySQLBackendVersionChoice(ALLOWED_MYSQL_VERSIONS),
     default=CacheableDefault(ALLOWED_MYSQL_VERSIONS[0]),
     envvar="MYSQL_VERSION",
     show_default=True,
@@ -269,7 +270,7 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option,
 option_postgres_version = click.option(
     "-P",
     "--postgres-version",
-    type=CacheableChoice(ALLOWED_POSTGRES_VERSIONS),
+    type=BackendVersionChoice(ALLOWED_POSTGRES_VERSIONS),
     default=CacheableDefault(ALLOWED_POSTGRES_VERSIONS[0]),
     envvar="POSTGRES_VERSION",
     show_default=True,

dev/breeze/src/airflow_breeze/utils/custom_param_types.py

@@ -193,7 +193,27 @@ def __init__(self, choices, case_sensitive: bool = True) -> None:
         super().__init__(choices=choices, case_sensitive=case_sensitive)
 
 
-class MySQLBackendVersionType(CacheableChoice):
+class BackendVersionChoice(CacheableChoice):
+    """
+    This specialized type of parameter allows to override the value of parameter with the BACKEND_VERSION
+    environment variable if it is set.
+
+    It's used to pass single matrix element in the matrix of tests when we run tests in CI - so that we do
+    not have to pass different matrices for different backends (which will be used to get the workflows
+    muvh more DRY).
+    """
+
+    name = "BackendVersionChoice"
+
+    def convert(self, value: Any, param: Parameter | None, ctx: Context | None) -> Any:
+        backend_version_env_value = os.environ.get("BACKEND_VERSION")
+        if backend_version_env_value:
+            if backend_version_env_value in self.choices:
+                value = backend_version_env_value
+        return super().convert(value, param, ctx)
+
+
+class MySQLBackendVersionChoice(BackendVersionChoice):
     def convert(self, value, param, ctx):
         if isinstance(value, CacheableDefault):
             param_name = param.envvar if param.envvar else param.name.upper()

dev/breeze/src/airflow_breeze/utils/selective_checks.py

@@ -516,7 +516,7 @@ def postgres_exclude(self) -> list[dict[str, str]]:
             return []
         return [
             # Exclude all combinations that are repeating python/postgres versions
-            {"python-version": python_version, "postgres-version": postgres_version}
+            {"python-version": python_version, "backend-version": postgres_version}
             for python_version, postgres_version in excluded_combos(
                 CURRENT_PYTHON_MAJOR_MINOR_VERSIONS, CURRENT_POSTGRES_VERSIONS
             )
@@ -529,7 +529,7 @@ def mysql_exclude(self) -> list[dict[str, str]]:
             return []
         return [
             # Exclude all combinations that are repeating python/mysql versions
-            {"python-version": python_version, "mysql-version": mysql_version}
+            {"python-version": python_version, "backend-version": mysql_version}
             for python_version, mysql_version in excluded_combos(
                 CURRENT_PYTHON_MAJOR_MINOR_VERSIONS, CURRENT_MYSQL_VERSIONS
             )

dev/breeze/src/airflow_breeze/utils/visuals.py

@@ -128,7 +128,7 @@
 
         You can disable this cheatsheet by running:
 
-            breeze setup config --no-cheatsheet
+            breeze setup config --no-cheatshee
 
 """
 CHEATSHEET_STYLE = "white"

scripts/docker/entrypoint_ci.sh

@@ -98,19 +98,19 @@ function environment_initialization() {
     if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} == "true" ]]; then
         return
     fi
-    if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
-        if [[ ${BACKEND:=} == "mssql" ]]; then
-            echo "${COLOR_RED}ARM platform is not supported for ${BACKEND} backend. Exiting.${COLOR_RESET}"
-            exit 1
-        fi
-    fi
-
     echo
     echo "${COLOR_BLUE}Running Initialization. Your basic configuration is:${COLOR_RESET}"
     echo
     echo "  * ${COLOR_BLUE}Airflow home:${COLOR_RESET} ${AIRFLOW_HOME}"
     echo "  * ${COLOR_BLUE}Airflow sources:${COLOR_RESET} ${AIRFLOW_SOURCES}"
-    echo "  * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}"
+    echo "  * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:=}"
+    if [[ ${BACKEND=} == "postgres" ]]; then
+        echo "  * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Postgres: ${POSTGRES_VERSION}"
+    elif [[ ${BACKEND=} == "mysql" ]]; then
+        echo "  * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} MySQL: ${MYSQL_VERSION}"
+    elif [[ ${BACKEND=} == "sqlite" ]]; then
+        echo "  * ${COLOR_BLUE}Airflow backend:${COLOR_RESET} Sqlite"
+    fi
     echo
 
     if [[ ${STANDALONE_DAG_PROCESSOR=} == "true" ]]; then