Sync v2-9-stable with v2-9-test to release python client v2.9.3

.github/workflows/additional-ci-image-checks.yml

@@ -145,26 +145,26 @@ jobs:
       - name: "Check that image builds quickly"
         run: breeze shell --max-time 600 --platform "linux/amd64"
 
-  # This is only a check if ARM images are successfully building when committer runs PR from
-  # Apache repository. This is needed in case you want to fix failing cache job in "canary" run
-  # There is no point in running this one in "canary" run, because the above step is doing the
-  # same build anyway.
-  build-ci-arm-images:
-    name: Build CI ARM images (in-workflow)
-    uses: ./.github/workflows/ci-image-build.yml
-    permissions:
-      contents: read
-      packages: write
-    secrets: inherit
-    with:
-      push-image: "false"
-      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
-      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
-      image-tag: ${{ inputs.image-tag }}
-      python-versions: ${{ inputs.python-versions }}
-      platform: "linux/arm64"
-      branch: ${{ inputs.branch }}
-      constraints-branch: ${{ inputs.constraints-branch }}
-      use-uv: "true"
-      upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
-      docker-cache: ${{ inputs.docker-cache }}
+#  # This is only a check if ARM images are successfully building when committer runs PR from
+#  # Apache repository. This is needed in case you want to fix failing cache job in "canary" run
+#  # There is no point in running this one in "canary" run, because the above step is doing the
+#  # same build anyway.
+#  build-ci-arm-images:
+#    name: Build CI ARM images (in-workflow)
+#    uses: ./.github/workflows/ci-image-build.yml
+#    permissions:
+#      contents: read
+#      packages: write
+#    secrets: inherit
+#    with:
+#      push-image: "false"
+#      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
+#      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
+#      image-tag: ${{ inputs.image-tag }}
+#      python-versions: ${{ inputs.python-versions }}
+#      platform: "linux/arm64"
+#      branch: ${{ inputs.branch }}
+#      constraints-branch: ${{ inputs.constraints-branch }}
+#      use-uv: "true"
+#      upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
+#      docker-cache: ${{ inputs.docker-cache }}

.github/workflows/ci.yml

@@ -19,11 +19,12 @@
 name: Tests
 on:  # yamllint disable-line rule:truthy
   schedule:
-    - cron: '28 0 * * *'
+    - cron: '28 7,15 * * *'
   push:
-    branches: ['main', 'v[0-9]+-[0-9]+-test']
+    branches: ['v[0-9]+-[0-9]+-test']
   pull_request:
     branches: ['main']
+  workflow_dispatch:
 permissions:
   # All other permissions are set to none
   contents: read

.github/workflows/finalize-tests.yml

@@ -150,28 +150,28 @@ jobs:
       docker-cache: ${{ inputs.docker-cache }}
     if: inputs.canary-run == 'true'
 
-  push-buildx-cache-to-github-registry-arm:
-    name: Push Regular ARM Image Cache
-    needs: [update-constraints]
-    uses: ./.github/workflows/push-image-cache.yml
-    permissions:
-      contents: read
-      packages: write
-    secrets: inherit
-    with:
-      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
-      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
-      cache-type: "Regular ARM"
-      include-prod-images: "true"
-      push-latest-images: "true"
-      platform: "linux/arm64"
-      python-versions: ${{ inputs.python-versions }}
-      branch: ${{ inputs.branch }}
-      constraints-branch: ${{ inputs.constraints-branch }}
-      use-uv: "true"
-      include-success-outputs: ${{ inputs.include-success-outputs }}
-      docker-cache: ${{ inputs.docker-cache }}
-    if: inputs.canary-run == 'true'
+  #  push-buildx-cache-to-github-registry-arm:
+  #    name: Push Regular ARM Image Cache
+  #    needs: [update-constraints]
+  #    uses: ./.github/workflows/push-image-cache.yml
+  #    permissions:
+  #      contents: read
+  #      packages: write
+  #    secrets: inherit
+  #    with:
+  #      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
+  #      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
+  #      cache-type: "Regular ARM"
+  #      include-prod-images: "true"
+  #      push-latest-images: "true"
+  #      platform: "linux/arm64"
+  #      python-versions: ${{ inputs.python-versions }}
+  #      branch: ${{ inputs.branch }}
+  #      constraints-branch: ${{ inputs.constraints-branch }}
+  #      use-uv: "true"
+  #      include-success-outputs: ${{ inputs.include-success-outputs }}
+  #      docker-cache: ${{ inputs.docker-cache }}
+  #    if: inputs.canary-run == 'true'
 
   summarize-warnings:
     timeout-minutes: 15

.github/workflows/static-checks-mypy-docs.yml

@@ -214,41 +214,41 @@ jobs:
       - name: "Building docs with ${{ matrix.flag }} flag"
         run: >
           breeze build-docs ${{ inputs.docs-list-as-string }} ${{ matrix.flag }}
-      - name: "Clone airflow-site"
-        run: >
-          git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site &&
-          echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV"
-        if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
-      - name: "Publish docs"
-        run: >
-          breeze release-management publish-docs --override-versioned --run-in-parallel
-          ${{ inputs.docs-list-as-string }}
-        if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
-      - name: "Generate back references for providers"
-        run: breeze release-management add-back-references all-providers
-        if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
-      - name: "Generate back references for apache-airflow"
-        run: breeze release-management add-back-references apache-airflow
-        if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
-      - name: "Generate back references for docker-stack"
-        run: breeze release-management add-back-references docker-stack
-        if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
-      - name: "Generate back references for helm-chart"
-        run: breeze release-management add-back-references helm-chart
-        if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a  # v4.0.1
-        if: >
-          inputs.canary-run == 'true' &&
-          inputs.branch == 'main' &&
-          matrix.flag == '--docs-only'
-        with:
-          aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-central-1
-      - name: "Upload documentation to AWS S3"
-        if: >
-          inputs.canary-run == 'true' &&
-          inputs.branch == 'main' &&
-          matrix.flag == '--docs-only'
-        run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs
+      #      - name: "Clone airflow-site"
+      #        run: >
+      #          git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site &&
+      #          echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV"
+      #        if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+      #      - name: "Publish docs"
+      #        run: >
+      #          breeze release-management publish-docs --override-versioned --run-in-parallel
+      #          ${{ inputs.docs-list-as-string }}
+      #        if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+      #      - name: "Generate back references for providers"
+      #        run: breeze release-management add-back-references all-providers
+      #        if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+      #      - name: "Generate back references for apache-airflow"
+      #        run: breeze release-management add-back-references apache-airflow
+      #        if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+      #      - name: "Generate back references for docker-stack"
+      #        run: breeze release-management add-back-references docker-stack
+      #        if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+      #      - name: "Generate back references for helm-chart"
+      #        run: breeze release-management add-back-references helm-chart
+      #        if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+      #      - name: Configure AWS credentials
+      #        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a  # v4.0.1
+      #        if: >
+      #          inputs.canary-run == 'true' &&
+      #          inputs.branch == 'main' &&
+      #          matrix.flag == '--docs-only'
+      #        with:
+      #          aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
+      #          aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
+      #          aws-region: eu-central-1
+      #      - name: "Upload documentation to AWS S3"
+      #        if: >
+      #          inputs.canary-run == 'true' &&
+      #          inputs.branch == 'main' &&
+      #          matrix.flag == '--docs-only'
+      #        run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs

clients/python/README.md

@@ -240,9 +240,10 @@ fulfilling the request.
 
 This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
 
-- API version: 2.8.0
-- Package version: 2.8.0
+- API version: 2.9.0
+- Package version: 2.9.0
 - Build package: org.openapitools.codegen.languages.PythonClientCodegen
+
 For more information, please visit [https://airflow.apache.org](https://airflow.apache.org)
 
 ## Requirements.

dev/breeze/src/airflow_breeze/commands/ci_commands.py

@@ -373,6 +373,10 @@ def workflow_info(context: str) -> WorkflowInfo:
         target_repo = ctx["repository"]
         head_repo = ctx["repository"]
         event_name = ctx["event_name"]
+    elif event_name == "workflow_dispatch":
+        target_repo = ctx["repository"]
+        head_repo = ctx["repository"]
+        event_name = ctx["event_name"]
     elif event_name == "pull_request_target":
         target_repo = ctx["repository"]
         head_repo = ctx["repository"]

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -2697,7 +2697,11 @@ def split_version_and_suffix(file_name: str, suffix: str) -> VersionedFile:
 TARGET_API_YAML_PATH = PYTHON_CLIENT_DIR_PATH / "v1.yaml"
 OPENAPI_GENERATOR_CLI_VER = "5.4.0"
 
-GENERATED_CLIENT_DIRECTORIES_TO_COPY = ["airflow_client", "docs", "test"]
+GENERATED_CLIENT_DIRECTORIES_TO_COPY: list[Path] = [
+    Path("airflow_client") / "client",
+    Path("docs"),
+    Path("test"),
+]
 FILES_TO_COPY_TO_CLIENT_REPO = [
     ".gitignore",
     ".openapi-generator-ignore",

dev/breeze/src/airflow_breeze/global_constants.py

@@ -33,9 +33,11 @@
 RUNS_ON_PUBLIC_RUNNER = '["ubuntu-22.04"]'
 # we should get more sophisticated logic here in the future, but for now we just check if
 # we use self airflow, vm-based, amd hosted runner as a default
+# TODO: temporarily we need to switch to public runners to avoid issues with self-hosted runners
+RUNS_ON_SELF_HOSTED_RUNNER = '["ubuntu-22.04"]'
 # TODO: when we have it properly set-up with labels we should change it to
 # RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "airflow-runner", "vm-runner", "X64"]'
-RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "Linux", "X64"]'
+# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "Linux", "X64"]'
 SELF_HOSTED_RUNNERS_CPU_COUNT = 8
 
 ANSWER = ""

dev/breeze/tests/test_pr_info.py

@@ -81,7 +81,9 @@ def test_runs_on_self_hosted():
         assert wi.head_repo == "apache/airflow"
         assert wi.event_name == "pull_request"
         assert wi.pr_number == 1234
-        assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
+        # TODO: fix it when we fix self-hosted runners
+        assert wi.get_runs_on() == '["ubuntu-22.04"]'
+        # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
         assert wi.is_canary_run() == "false"
         assert wi.run_coverage() == "false"
 
@@ -123,7 +125,9 @@ def test_runs_on_push_other_branch():
         assert wi.head_repo == "apache/airflow"
         assert wi.event_name == "push"
         assert wi.pr_number is None
-        assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
+        # TODO: revert it when we fix self-hosted runners
+        assert wi.get_runs_on() == '["ubuntu-22.04"]'
+        # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
         assert wi.is_canary_run() == "false"
         assert wi.run_coverage() == "false"
 
@@ -137,6 +141,8 @@ def test_runs_on_push_v_test_branch():
         assert wi.head_repo == "apache/airflow"
         assert wi.event_name == "push"
         assert wi.pr_number is None
-        assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
+        # TODO: revert it when we fix self-hosted runners
+        assert wi.get_runs_on() == '["ubuntu-22.04"]'
+        # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
         assert wi.is_canary_run() == "true"
         assert wi.run_coverage() == "false"

dev/breeze/tests/test_selective_checks.py

@@ -1636,13 +1636,20 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
             "apache/airflow",
             (),
             dict(),
-            '["self-hosted", "Linux", "X64"]',
-            "true",
-            "true",
-            "true",
+            # TODO: revert it when we fix self-hosted runners
+            '["ubuntu-22.04"]',
+            # '["self-hosted", "Linux", "X64"]',
+            # TODO: revert it when we fix self-hosted runners
+            "false",
             "false",
+            # "true",
+            # "true",
             "true",
             "false",
+            # TODO: revert it when we fix self-hosted runners
+            "false",
+            # "true",
+            "false",
             False,
             id="Push event",
         ),
@@ -1700,13 +1707,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
             "apache/airflow",
             (),
             dict(),
-            '["self-hosted", "Linux", "X64"]',
-            "true",
-            "true",
-            "true",
+            '["ubuntu-22.04"]',
+            "false",
             "false",
             "true",
             "false",
+            "false",
+            "false",
             False,
             id="Pull request committer",
         ),
@@ -1780,13 +1787,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
             "apache/airflow",
             [],
             dict(),
-            '["self-hosted", "Linux", "X64"]',
-            "true",
-            "true",
-            "true",
+            '["ubuntu-22.04"]',
+            "false",
             "false",
             "true",
             "false",
+            "false",
+            "false",
             False,
             id="Pull request target committer",
         ),
@@ -2065,7 +2072,7 @@ def test_mypy_matches(
             ("README.md",),
             {
                 "is-committer-build": "true",
-                "runs-on-as-json-default": '["self-hosted", "Linux", "X64"]',
+                "runs-on-as-json-default": '["ubuntu-22.04"]',
             },
             "potiuk",
             (),
@@ -2075,7 +2082,7 @@ def test_mypy_matches(
             ("README.md",),
             {
                 "is-committer-build": "false",
-                "runs-on-as-json-default": '["self-hosted", "Linux", "X64"]',
+                "runs-on-as-json-default": '["ubuntu-22.04"]',
             },
             "potiuk",
             ("non committer build",),