Run unit tests for Providers with airflow installed as package. (apac…

…he#39513)

This PR adds the option of running unit tests for providers against
a specific airflow version (for example released version in PyPI)
and enables it for back-compatibility testing for 2.9.1. In the
future it could be used to run forward-compatibility testing with
Airflow 3 as well.

.github/workflows/check-providers.yml

@@ -43,7 +43,11 @@ on:  # yamllint disable-line rule:truthy
       providers-compatibility-checks:
         description: >
           JSON-formatted array of providers compatibility checks in the form of array of dicts
-          (airflow-version, python-versions, remove-providers)
+          (airflow-version, python-versions, remove-providers, run-tests)
+        required: true
+        type: string
+      providers-test-types-list-as-string:
+        description: "List of parallel provider test types as string"
         required: true
         type: string
       skip-provider-tests:
@@ -237,6 +241,9 @@ jobs:
       - name: >
           Install and verify all provider packages and airflow on
           Airflow ${{ matrix.airflow-version }}:Python ${{ matrix.python-version }}
+        # We do not need to run import check if we run tests, the tests should cover all the import checks
+        # automatically
+        if: matrix.run-tests != 'true'
         run: >
           breeze release-management verify-provider-packages
           --use-packages-from-dist
@@ -245,3 +252,17 @@ jobs:
           --airflow-constraints-reference constraints-${{matrix.airflow-version}}
           --providers-skip-constraints
           --install-airflow-with-constraints
+      - name: >
+          Run provider unit tests on
+          Airflow ${{ matrix.airflow-version }}:Python ${{ matrix.python-version }}
+        if: matrix.run-tests == 'true'
+        run: >
+          breeze testing tests --run-in-parallel
+          --parallel-test-types "${{ inputs.providers-test-types-list-as-string }}"
+          --use-packages-from-dist
+          --package-format wheel
+          --use-airflow-version "${{ matrix.airflow-version }}"
+          --airflow-constraints-reference constraints-${{matrix.airflow-version}}
+          --install-airflow-with-constraints
+          --providers-skip-constraints
+          --skip-providers "${{ matrix.remove-providers }}"

.github/workflows/ci.yml

@@ -77,6 +77,8 @@ jobs:
       full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }}
       parallel-test-types-list-as-string: >-
         ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }}
+      providers-test-types-list-as-string: >-
+        ${{ steps.selective-checks.outputs.providers-test-types-list-as-string }}
       include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }}
       postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
       mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
@@ -315,6 +317,7 @@ jobs:
       providers-compatibility-checks: ${{ needs.build-info.outputs.providers-compatibility-checks }}
       skip-provider-tests: ${{ needs.build-info.outputs.skip-provider-tests }}
       python-versions: ${{ needs.build-info.outputs.python-versions }}
+      providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}
 
   tests-helm:
     name: "Helm tests"

Dockerfile.ci

@@ -984,6 +984,8 @@ function determine_airflow_to_use() {
         mkdir -p "${AIRFLOW_SOURCES}"/tmp/
     else
         python "${IN_CONTAINER_DIR}/install_airflow_and_providers.py"
+        # Some packages might leave legacy typing module which causes test issues
+        pip uninstall -y typing || true
     fi
 
     if [[ "${USE_AIRFLOW_VERSION}" =~ ^2\.2\..*|^2\.1\..*|^2\.0\..* && "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=}" != "" ]]; then

airflow/providers/openlineage/conf.py

@@ -100,7 +100,6 @@ def is_disabled() -> bool:
     option = os.getenv("OPENLINEAGE_DISABLED", "")
     if _is_true(option):
         return True
-
     # Check if both 'transport' and 'config_path' are not present and also
     # if legacy 'OPENLINEAGE_URL' environment variables is not set
     return transport() == {} and config_path(True) == "" and os.getenv("OPENLINEAGE_URL", "") == ""

airflow/providers/openlineage/plugins/openlineage.py

@@ -39,3 +39,6 @@ class OpenLineageProviderPlugin(AirflowPlugin):
     if not conf.is_disabled():
         macros = [lineage_job_namespace, lineage_job_name, lineage_run_id, lineage_parent_id]
         listeners = [get_openlineage_listener()]
+    else:
+        macros = []
+        listeners = []