Split tests to core/providers/task-sdk/integration/system

The tests execution was traditionally using single "breeze testing
tests" command and you could select which test to run via TEST_TYPE.

However the recent move of providers and adding task_sdk necessitates
splitting the tests commands into separate commands for core, providers,
task_sdk, helm, integration and system.

This is done via introducing "TEST_GROUP" - which determines which
group of tests is being executed, and dedicated testing command for each
of the groups - with "db" and "non-db" variants where applicable.

Cleanup and small refactoring has been done to make it easier to
reason about parameters passed down from the command line to
docker and in-container pytest command.

Related: #42632

.github/workflows/check-providers.yml

@@ -40,7 +40,7 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to upgrade to newer dependencies"
         required: true
         type: string
-      affected-providers-list-as-string:
+      selected-providers-list-as-string:
         description: "List of affected providers as string"
         required: false
         type: string
@@ -163,7 +163,7 @@ jobs:
         run: >
           breeze release-management prepare-provider-packages --include-not-ready-providers
           --version-suffix-for-pypi dev0 --package-format sdist
-          ${{ inputs.affected-providers-list-as-string }}
+          ${{ inputs.selected-providers-list-as-string }}
       - name: "Prepare airflow package: sdist"
         run: >
           breeze release-management prepare-airflow-package
@@ -187,7 +187,7 @@ jobs:
           --providers-constraints-location
           /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
           --run-in-parallel
-        if: inputs.affected-providers-list-as-string == ''
+        if: inputs.selected-providers-list-as-string == ''
       - name: "Install affected provider packages and airflow via sdist files"
         run: >
           breeze release-management install-provider-packages
@@ -198,7 +198,7 @@ jobs:
           --providers-constraints-location
           /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
           --run-in-parallel
-        if: inputs.affected-providers-list-as-string != ''
+        if: inputs.selected-providers-list-as-string != ''
 
   providers-compatibility-checks:
     timeout-minutes: 80
@@ -268,7 +268,7 @@ jobs:
           Airflow ${{ matrix.airflow-version }}:Python ${{ matrix.python-version }}
         if: matrix.run-tests == 'true'
         run: >
-          breeze testing tests --run-in-parallel
+          breeze testing providers-tests --run-in-parallel
           --parallel-test-types "${{ inputs.providers-test-types-list-as-string }}"
           --use-packages-from-dist
           --package-format wheel

.github/workflows/ci.yml

@@ -58,8 +58,8 @@ jobs:
       image-tag: ${{ github.event.pull_request.head.sha || github.sha }}
       docker-cache: ${{ steps.selective-checks.outputs.docker-cache }}
       disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }}
-      affected-providers-list-as-string: >-
-        ${{ steps.selective-checks.outputs.affected-providers-list-as-string }}
+      selected-providers-list-as-string: >-
+        ${{ steps.selective-checks.outputs.selected-providers-list-as-string }}
       upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
       python-versions: ${{ steps.selective-checks.outputs.python-versions }}
       python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
@@ -79,12 +79,12 @@ jobs:
       default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
       force-pip: ${{ steps.selective-checks.outputs.force-pip }}
       full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }}
-      parallel-test-types-list-as-string: >-
-        ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }}
+      core-test-types-list-as-string: >-
+        ${{ steps.selective-checks.outputs.core-test-types-list-as-string }}
       providers-test-types-list-as-string: >-
         ${{ steps.selective-checks.outputs.providers-test-types-list-as-string }}
-      separate-test-types-list-as-string: >-
-        ${{ steps.selective-checks.outputs.separate-test-types-list-as-string }}
+      individual-providers-test-types-list-as-string: >-
+        ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-string }}
       include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }}
       postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
       mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
@@ -135,6 +135,7 @@ jobs:
       testable-integrations: ${{ steps.selective-checks.outputs.testable-integrations }}
       canary-run: ${{ steps.source-run-info.outputs.canary-run }}
       run-coverage: ${{ steps.source-run-info.outputs.run-coverage }}
+      test-groups: ${{ steps.source-run-info.outputs.test-groups }}
     steps:
       - name: "Cleanup repo"
         shell: bash
@@ -336,7 +337,7 @@ jobs:
       canary-run: ${{ needs.build-info.outputs.canary-run }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
-      affected-providers-list-as-string: ${{ needs.build-info.outputs.affected-providers-list-as-string }}
+      selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }}
       providers-compatibility-checks: ${{ needs.build-info.outputs.providers-compatibility-checks }}
       skip-provider-tests: ${{ needs.build-info.outputs.skip-provider-tests }}
       python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -362,7 +363,7 @@ jobs:
       needs.build-info.outputs.latest-versions-only != 'true'
 
   tests-postgres:
-    name: "Postgres tests"
+    name: "Postgres core tests"
     uses: ./.github/workflows/run-unit-tests.yml
     needs: [build-info, wait-for-ci-images]
     permissions:
@@ -374,20 +375,22 @@ jobs:
       backend: "postgres"
       test-name: "Postgres"
       test-scope: "DB"
+      test-groups: ${{ needs.build-info.outputs.test-groups }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       backend-versions: ${{ needs.build-info.outputs.postgres-versions }}
       excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
       excludes: ${{ needs.build-info.outputs.postgres-exclude }}
-      parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }}
+      core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
+      providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       run-migration-tests: "true"
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'
+    if: needs.build-info.outputs.run-tests == 'true'
 
   tests-mysql:
-    name: "MySQL tests"
+    name: "MySQL core tests"
     uses: ./.github/workflows/run-unit-tests.yml
     needs: [build-info, wait-for-ci-images]
     permissions:
@@ -399,20 +402,22 @@ jobs:
       backend: "mysql"
       test-name: "MySQL"
       test-scope: "DB"
+      test-groups: ${{ needs.build-info.outputs.test-groups }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       backend-versions: ${{ needs.build-info.outputs.mysql-versions }}
       excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
       excludes: ${{ needs.build-info.outputs.mysql-exclude }}
-      parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }}
+      core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
+      providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       run-migration-tests: "true"
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'
+    if: needs.build-info.outputs.run-tests == 'true'
 
   tests-sqlite:
-    name: "Sqlite tests"
+    name: "Sqlite core tests"
     uses: ./.github/workflows/run-unit-tests.yml
     needs: [build-info, wait-for-ci-images]
     permissions:
@@ -425,21 +430,23 @@ jobs:
       test-name: "Sqlite"
       test-name-separator: ""
       test-scope: "DB"
+      test-groups: ${{ needs.build-info.outputs.test-groups }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       # No versions for sqlite
       backend-versions: "['']"
       excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
       excludes: ${{ needs.build-info.outputs.sqlite-exclude }}
-      parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }}
+      core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
+      providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       run-migration-tests: "true"
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'
+    if: needs.build-info.outputs.run-tests == 'true'
 
   tests-non-db:
-    name: "Non-DB tests"
+    name: "Non-DB core tests"
     uses: ./.github/workflows/run-unit-tests.yml
     needs: [build-info, wait-for-ci-images]
     permissions:
@@ -452,17 +459,19 @@ jobs:
       test-name: ""
       test-name-separator: ""
       test-scope: "Non-DB"
+      test-groups: ${{ needs.build-info.outputs.test-groups }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       # No versions for non-db
       backend-versions: "['']"
       excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
       excludes: ${{ needs.build-info.outputs.sqlite-exclude }}
-      parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }}
+      core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
+      providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-    if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.only-new-ui-files != 'true'
+    if: needs.build-info.outputs.run-tests == 'true'
 
   tests-special:
     name: "Special tests"
@@ -478,9 +487,11 @@ jobs:
        needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' ||
        needs.build-info.outputs.full-tests-needed == 'true')
     with:
+      test-groups: ${{ needs.build-info.outputs.test-groups }}
       runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
-      parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }}
+      core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
+      providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       python-versions: ${{ needs.build-info.outputs.python-versions }}
@@ -511,7 +522,7 @@ jobs:
     if: needs.build-info.outputs.run-tests == 'true'
 
   tests-with-lowest-direct-resolution:
-    name: "Lowest direct dependency resolution tests"
+    name: "Lowest direct dependency providers tests"
     needs: [build-info, wait-for-ci-images]
     uses: ./.github/workflows/run-unit-tests.yml
     permissions:
@@ -525,13 +536,16 @@ jobs:
       test-name: "LowestDeps-Postgres"
       force-lowest-dependencies: "true"
       test-scope: "All"
+      test-groups: ${{ needs.build-info.outputs.test-groups }}
       backend: "postgres"
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       backend-versions: "['${{ needs.build-info.outputs.default-postgres-version }}']"
       excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }}
       excludes: "[]"
-      parallel-test-types-list-as-string: ${{ needs.build-info.outputs.separate-test-types-list-as-string }}
+      core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
+      # yamllint disable rule:line-length
+      providers-test-types-list-as-string: ${{ needs.build-info.outputs.individual-providers-test-types-list-as-string }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}

.github/workflows/helm-tests.yml

@@ -75,7 +75,7 @@ jobs:
       - name: "Prepare breeze & CI image: ${{inputs.default-python-version}}:${{inputs.image-tag}}"
         uses: ./.github/actions/prepare_breeze_and_image
       - name: "Helm Unit Tests: ${{ matrix.helm-test-package }}"
-        run: breeze testing helm-tests --helm-test-package "${{ matrix.helm-test-package }}"
+        run: breeze testing helm-tests --test-type "${{ matrix.helm-test-package }}"
 
   tests-helm-release:
     timeout-minutes: 80

.github/workflows/integration-tests.yml

@@ -28,8 +28,12 @@ on:  # yamllint disable-line rule:truthy
         description: "Tag to set for the image"
         required: true
         type: string
-      testable-integrations:
-        description: "The list of testable integrations as JSON array."
+      testable-core-integrations:
+        description: "The list of testable core integrations as JSON array."
+        required: true
+        type: string
+      testable-providers-integrations:
+        description: "The list of testable providers integrations as JSON array."
         required: true
         type: string
       default-postgres-version:
@@ -57,21 +61,67 @@ on:  # yamllint disable-line rule:truthy
         required: true
         type: string
 jobs:
-  tests-integration:
+  tests-core-integration:
+    timeout-minutes: 130
+    if: inputs.testable-core-integrations != '[]'
+    name: "Integration Tests: core ${{ matrix.integration }}"
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+    strategy:
+      fail-fast: false
+      matrix:
+        integration: ${{ fromJSON(inputs.testable-core-integrations) }}
+    env:
+      IMAGE_TAG: "${{ inputs.image-tag }}"
+      BACKEND: "postgres"
+      BACKEND_VERSION: ${{ inputs.default-postgres-version }}"
+      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+      JOB_ID: "integration-core-${{ matrix.integration }}"
+      SKIP_PROVIDER_TESTS: "${{ inputs.skip-provider-tests }}"
+      ENABLE_COVERAGE: "${{ inputs.run-coverage}}"
+      DEBUG_RESOURCES: "${{ inputs.debug-resources }}"
+      GITHUB_REPOSITORY: ${{ github.repository }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GITHUB_USERNAME: ${{ github.actor }}
+      VERBOSE: "true"
+    steps:
+      - name: "Cleanup repo"
+        shell: bash
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: "Cleanup docker"
+        run: ./scripts/ci/cleanup_docker.sh
+      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}"
+        uses: ./.github/actions/prepare_breeze_and_image
+      - name: "Integration Tests: core ${{ matrix.integration }}"
+        # yamllint disable rule:line-length
+        run: ./scripts/ci/testing/run_integration_tests_with_retry.sh core "${{ matrix.integration }}"
+      - name: "Post Tests success"
+        uses: ./.github/actions/post_tests_success
+        with:
+          codecov-token: ${{ secrets.CODECOV_TOKEN }}
+          python-version: ${{ inputs.default-python-version }}
+      - name: "Post Tests failure"
+        uses: ./.github/actions/post_tests_failure
+        if: failure()
+
+  tests-providers-integration:
     timeout-minutes: 130
-    if: inputs.testable-integrations != '[]'
-    name: "Integration Tests: ${{ matrix.integration }}"
+    if: inputs.testable-providers-integrations != '[]'
+    name: "Integration Tests: providers ${{ matrix.integration }}"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     strategy:
       fail-fast: false
       matrix:
-        integration: ${{ fromJSON(inputs.testable-integrations) }}
+        integration: ${{ fromJSON(inputs.testable-providers-integrations) }}
     env:
       IMAGE_TAG: "${{ inputs.image-tag }}"
       BACKEND: "postgres"
       BACKEND_VERSION: ${{ inputs.default-postgres-version }}"
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
-      JOB_ID: "integration-${{ matrix.integration }}"
+      JOB_ID: "integration-providers-${{ matrix.integration }}"
       SKIP_PROVIDER_TESTS: "${{ inputs.skip-provider-tests }}"
       ENABLE_COVERAGE: "${{ inputs.run-coverage}}"
       DEBUG_RESOURCES: "${{ inputs.debug-resources }}"
@@ -91,13 +141,14 @@ jobs:
         run: ./scripts/ci/cleanup_docker.sh
       - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}"
         uses: ./.github/actions/prepare_breeze_and_image
-      - name: "Integration Tests: ${{ matrix.integration }}"
-        run: ./scripts/ci/testing/run_integration_tests_with_retry.sh ${{ matrix.integration }}
-      - name: "Post Tests success: Integration Tests ${{ matrix.integration }}"
+      - name: "Integration Tests: providers ${{ matrix.integration }}"
+        # yamllint disable rule:line-length
+        run: ./scripts/ci/testing/run_integration_tests_with_retry.sh providers "${{ matrix.integration }}"
+      - name: "Post Tests success"
         uses: ./.github/actions/post_tests_success
         with:
           codecov-token: ${{ secrets.CODECOV_TOKEN }}
           python-version: ${{ inputs.default-python-version }}
-      - name: "Post Tests failure: Integration Tests ${{ matrix.integration }}"
+      - name: "Post Tests failure"
         uses: ./.github/actions/post_tests_failure
         if: failure()