From 71475fa0ae9acf5c90fef1091fb1dfb67c169b96 Mon Sep 17 00:00:00 2001 From: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> Date: Thu, 2 May 2024 18:07:28 -0400 Subject: [PATCH 1/5] Update python client version in README (#39384) (cherry picked from commit 495814e908890763d9429df7ad9f4f33908fef98) --- clients/python/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/clients/python/README.md b/clients/python/README.md index dc8fd402ae2bb..f0964d555276f 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -240,9 +240,10 @@ fulfilling the request. This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project: -- API version: 2.8.0 -- Package version: 2.8.0 +- API version: 2.9.0 +- Package version: 2.9.0 - Build package: org.openapitools.codegen.languages.PythonClientCodegen + For more information, please visit [https://airflow.apache.org](https://airflow.apache.org) ## Requirements. From 9aa67123fb13f0887231763af7d833da714e6ea2 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 23 Apr 2024 00:41:11 +0200 Subject: [PATCH 2/5] Only replace "client" subdir when copying sources to python client (#39181) When the client is generated, only the "client" subdirectory should be replaced rather than "airflow_client" parent so that README and gitignores are not removed. (cherry picked from commit 57995d098e7051d0be6cfa6e614eb898261237f7) --- .../airflow_breeze/commands/release_management_commands.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index cc5bf20cb0625..228a49d7b9807 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -2697,7 +2697,11 @@ def split_version_and_suffix(file_name: str, suffix: str) -> VersionedFile: TARGET_API_YAML_PATH = PYTHON_CLIENT_DIR_PATH / "v1.yaml" OPENAPI_GENERATOR_CLI_VER = "5.4.0" -GENERATED_CLIENT_DIRECTORIES_TO_COPY = ["airflow_client", "docs", "test"] +GENERATED_CLIENT_DIRECTORIES_TO_COPY: list[Path] = [ + Path("airflow_client") / "client", + Path("docs"), + Path("test"), +] FILES_TO_COPY_TO_CLIENT_REPO = [ ".gitignore", ".openapi-generator-ignore", From bacd04319eb987cc63078605a9853eb817ba7994 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 21 Jul 2024 15:57:46 +0200 Subject: [PATCH 3/5] Temporarily switch to public github runners for canary runs (#40913) We are having problems with self-hosted runners and until we fix the problems we switch main builds to only run on main with regular schedule 1x day to serve as a canary build. They will run longer than on self-hosted runners. We also add option to trigger the workflow manually from GitHub interface (by maintainers). This should cover most tests with the following limitations: * ARM builds and cache are not going to run (we can still refresh cache manually periodicaly to get cache for mac OS Breeze users * Releasing images via workflow will not work - we can still do it manually from preparing workstations (cherry picked from commit 3a00909933e76872be8fd4ef2e5e38ff29168d97) --- .../workflows/additional-ci-image-checks.yml | 46 +++++++++---------- .github/workflows/ci.yml | 5 +- .github/workflows/finalize-tests.yml | 44 +++++++++--------- .../airflow_breeze/commands/ci_commands.py | 4 ++ .../src/airflow_breeze/global_constants.py | 4 +- dev/breeze/tests/test_pr_info.py | 12 +++-- dev/breeze/tests/test_selective_checks.py | 15 ++++-- 7 files changed, 75 insertions(+), 55 deletions(-) diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 7d0eb88d24130..ae9efdb6b0340 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -145,26 +145,26 @@ jobs: - name: "Check that image builds quickly" run: breeze shell --max-time 600 --platform "linux/amd64" - # This is only a check if ARM images are successfully building when committer runs PR from - # Apache repository. This is needed in case you want to fix failing cache job in "canary" run - # There is no point in running this one in "canary" run, because the above step is doing the - # same build anyway. - build-ci-arm-images: - name: Build CI ARM images (in-workflow) - uses: ./.github/workflows/ci-image-build.yml - permissions: - contents: read - packages: write - secrets: inherit - with: - push-image: "false" - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} - image-tag: ${{ inputs.image-tag }} - python-versions: ${{ inputs.python-versions }} - platform: "linux/arm64" - branch: ${{ inputs.branch }} - constraints-branch: ${{ inputs.constraints-branch }} - use-uv: "true" - upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} - docker-cache: ${{ inputs.docker-cache }} +# # This is only a check if ARM images are successfully building when committer runs PR from +# # Apache repository. This is needed in case you want to fix failing cache job in "canary" run +# # There is no point in running this one in "canary" run, because the above step is doing the +# # same build anyway. +# build-ci-arm-images: +# name: Build CI ARM images (in-workflow) +# uses: ./.github/workflows/ci-image-build.yml +# permissions: +# contents: read +# packages: write +# secrets: inherit +# with: +# push-image: "false" +# runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} +# runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} +# image-tag: ${{ inputs.image-tag }} +# python-versions: ${{ inputs.python-versions }} +# platform: "linux/arm64" +# branch: ${{ inputs.branch }} +# constraints-branch: ${{ inputs.constraints-branch }} +# use-uv: "true" +# upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} +# docker-cache: ${{ inputs.docker-cache }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6d2a0714c155..82f01ea86305b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,11 +19,12 @@ name: Tests on: # yamllint disable-line rule:truthy schedule: - - cron: '28 0 * * *' + - cron: '28 7,15 * * *' push: - branches: ['main', 'v[0-9]+-[0-9]+-test'] + branches: ['v[0-9]+-[0-9]+-test'] pull_request: branches: ['main'] + workflow_dispatch: permissions: # All other permissions are set to none contents: read diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 5a3398d6a08fd..ba3ad5d6f36b1 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -150,28 +150,28 @@ jobs: docker-cache: ${{ inputs.docker-cache }} if: inputs.canary-run == 'true' - push-buildx-cache-to-github-registry-arm: - name: Push Regular ARM Image Cache - needs: [update-constraints] - uses: ./.github/workflows/push-image-cache.yml - permissions: - contents: read - packages: write - secrets: inherit - with: - runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} - runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} - cache-type: "Regular ARM" - include-prod-images: "true" - push-latest-images: "true" - platform: "linux/arm64" - python-versions: ${{ inputs.python-versions }} - branch: ${{ inputs.branch }} - constraints-branch: ${{ inputs.constraints-branch }} - use-uv: "true" - include-success-outputs: ${{ inputs.include-success-outputs }} - docker-cache: ${{ inputs.docker-cache }} - if: inputs.canary-run == 'true' + # push-buildx-cache-to-github-registry-arm: + # name: Push Regular ARM Image Cache + # needs: [update-constraints] + # uses: ./.github/workflows/push-image-cache.yml + # permissions: + # contents: read + # packages: write + # secrets: inherit + # with: + # runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + # runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} + # cache-type: "Regular ARM" + # include-prod-images: "true" + # push-latest-images: "true" + # platform: "linux/arm64" + # python-versions: ${{ inputs.python-versions }} + # branch: ${{ inputs.branch }} + # constraints-branch: ${{ inputs.constraints-branch }} + # use-uv: "true" + # include-success-outputs: ${{ inputs.include-success-outputs }} + # docker-cache: ${{ inputs.docker-cache }} + # if: inputs.canary-run == 'true' summarize-warnings: timeout-minutes: 15 diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py index 54f74f8026741..946a321852e4d 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py @@ -373,6 +373,10 @@ def workflow_info(context: str) -> WorkflowInfo: target_repo = ctx["repository"] head_repo = ctx["repository"] event_name = ctx["event_name"] + elif event_name == "workflow_dispatch": + target_repo = ctx["repository"] + head_repo = ctx["repository"] + event_name = ctx["event_name"] elif event_name == "pull_request_target": target_repo = ctx["repository"] head_repo = ctx["repository"] diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index c4a8dd59ae8ca..33ecdd5e61708 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -33,9 +33,11 @@ RUNS_ON_PUBLIC_RUNNER = '["ubuntu-22.04"]' # we should get more sophisticated logic here in the future, but for now we just check if # we use self airflow, vm-based, amd hosted runner as a default +# TODO: temporarily we need to switch to public runners to avoid issues with self-hosted runners +RUNS_ON_SELF_HOSTED_RUNNER = '["ubuntu-22.04"]' # TODO: when we have it properly set-up with labels we should change it to # RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "airflow-runner", "vm-runner", "X64"]' -RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "Linux", "X64"]' +# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "Linux", "X64"]' SELF_HOSTED_RUNNERS_CPU_COUNT = 8 ANSWER = "" diff --git a/dev/breeze/tests/test_pr_info.py b/dev/breeze/tests/test_pr_info.py index 3513cb56704c6..651009da0b5ec 100644 --- a/dev/breeze/tests/test_pr_info.py +++ b/dev/breeze/tests/test_pr_info.py @@ -81,7 +81,9 @@ def test_runs_on_self_hosted(): assert wi.head_repo == "apache/airflow" assert wi.event_name == "pull_request" assert wi.pr_number == 1234 - assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]' + # TODO: fix it when we fix self-hosted runners + assert wi.get_runs_on() == '["ubuntu-22.04"]' + # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]' assert wi.is_canary_run() == "false" assert wi.run_coverage() == "false" @@ -123,7 +125,9 @@ def test_runs_on_push_other_branch(): assert wi.head_repo == "apache/airflow" assert wi.event_name == "push" assert wi.pr_number is None - assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]' + # TODO: revert it when we fix self-hosted runners + assert wi.get_runs_on() == '["ubuntu-22.04"]' + # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]' assert wi.is_canary_run() == "false" assert wi.run_coverage() == "false" @@ -137,6 +141,8 @@ def test_runs_on_push_v_test_branch(): assert wi.head_repo == "apache/airflow" assert wi.event_name == "push" assert wi.pr_number is None - assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]' + # TODO: revert it when we fix self-hosted runners + assert wi.get_runs_on() == '["ubuntu-22.04"]' + # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]' assert wi.is_canary_run() == "true" assert wi.run_coverage() == "false" diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index ba6e12432c323..807f1ad9f4285 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1636,13 +1636,20 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "apache/airflow", (), dict(), - '["self-hosted", "Linux", "X64"]', - "true", - "true", - "true", + # TODO: revert it when we fix self-hosted runners + '["ubuntu-22.04"]', + # '["self-hosted", "Linux", "X64"]', + # TODO: revert it when we fix self-hosted runners + "false", "false", + # "true", + # "true", "true", "false", + # TODO: revert it when we fix self-hosted runners + "false", + # "true", + "false", False, id="Push event", ), From 5c90bc5e0c14675285e665a637a1d845a313c3d4 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 22 Jul 2024 08:55:12 +0200 Subject: [PATCH 4/5] Temporarily comment-out publishing docs in canary runs (#40921) There is not enough space to checkout airflow site on regular runners so until we fix self-hosted runners, we need to skip docs publishing. (cherry picked from commit 173fed0ce3e41f2ab730306cf21d39659b0a5234) --- .github/workflows/static-checks-mypy-docs.yml | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/.github/workflows/static-checks-mypy-docs.yml b/.github/workflows/static-checks-mypy-docs.yml index f517eb6a97815..0b057e6fbf53a 100644 --- a/.github/workflows/static-checks-mypy-docs.yml +++ b/.github/workflows/static-checks-mypy-docs.yml @@ -214,41 +214,41 @@ jobs: - name: "Building docs with ${{ matrix.flag }} flag" run: > breeze build-docs ${{ inputs.docs-list-as-string }} ${{ matrix.flag }} - - name: "Clone airflow-site" - run: > - git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site && - echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV" - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Publish docs" - run: > - breeze release-management publish-docs --override-versioned --run-in-parallel - ${{ inputs.docs-list-as-string }} - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Generate back references for providers" - run: breeze release-management add-back-references all-providers - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Generate back references for apache-airflow" - run: breeze release-management add-back-references apache-airflow - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Generate back references for docker-stack" - run: breeze release-management add-back-references docker-stack - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Generate back references for helm-chart" - run: breeze release-management add-back-references helm-chart - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - if: > - inputs.canary-run == 'true' && - inputs.branch == 'main' && - matrix.flag == '--docs-only' - with: - aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-central-1 - - name: "Upload documentation to AWS S3" - if: > - inputs.canary-run == 'true' && - inputs.branch == 'main' && - matrix.flag == '--docs-only' - run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs + # - name: "Clone airflow-site" + # run: > + # git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site && + # echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV" + # if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + # - name: "Publish docs" + # run: > + # breeze release-management publish-docs --override-versioned --run-in-parallel + # ${{ inputs.docs-list-as-string }} + # if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + # - name: "Generate back references for providers" + # run: breeze release-management add-back-references all-providers + # if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + # - name: "Generate back references for apache-airflow" + # run: breeze release-management add-back-references apache-airflow + # if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + # - name: "Generate back references for docker-stack" + # run: breeze release-management add-back-references docker-stack + # if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + # - name: "Generate back references for helm-chart" + # run: breeze release-management add-back-references helm-chart + # if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + # if: > + # inputs.canary-run == 'true' && + # inputs.branch == 'main' && + # matrix.flag == '--docs-only' + # with: + # aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} + # aws-region: eu-central-1 + # - name: "Upload documentation to AWS S3" + # if: > + # inputs.canary-run == 'true' && + # inputs.branch == 'main' && + # matrix.flag == '--docs-only' + # run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs From 70a928c1c2b824a8256ca1e7d8a9b492987cbab7 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Mon, 22 Jul 2024 18:18:36 +0530 Subject: [PATCH 5/5] Update breeze tests to use ubuntu-22.04 instead of self-hosted, Linux, X64 This change is only temporary to make the CI happy - https://github.com/apache/airflow/pull/40930\#issuecomment-2242813945 --- dev/breeze/tests/test_selective_checks.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 807f1ad9f4285..52259a896a1c8 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1707,13 +1707,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "apache/airflow", (), dict(), - '["self-hosted", "Linux", "X64"]', - "true", - "true", - "true", + '["ubuntu-22.04"]', + "false", "false", "true", "false", + "false", + "false", False, id="Pull request committer", ), @@ -1787,13 +1787,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d "apache/airflow", [], dict(), - '["self-hosted", "Linux", "X64"]', - "true", - "true", - "true", + '["ubuntu-22.04"]', + "false", "false", "true", "false", + "false", + "false", False, id="Pull request target committer", ), @@ -2072,7 +2072,7 @@ def test_mypy_matches( ("README.md",), { "is-committer-build": "true", - "runs-on-as-json-default": '["self-hosted", "Linux", "X64"]', + "runs-on-as-json-default": '["ubuntu-22.04"]', }, "potiuk", (), @@ -2082,7 +2082,7 @@ def test_mypy_matches( ("README.md",), { "is-committer-build": "false", - "runs-on-as-json-default": '["self-hosted", "Linux", "X64"]', + "runs-on-as-json-default": '["ubuntu-22.04"]', }, "potiuk", ("non committer build",),