This repository was archived by the owner on Sep 4, 2024. It is now read-only.
Deprecate the provider and proxy to upstream Airflow provider #262
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Build and test astro databricks provider | |
| on: # yamllint disable-line rule:truthy | |
| push: | |
| branches: [main, deprecate-provider] | |
| pull_request: | |
| branches: [main, 'release-**'] | |
| # Run on PRs from forks | |
| pull_request_target: | |
| branches: ['main'] | |
| types: ['labeled'] | |
| release: | |
| types: ['created'] | |
| # This allows a subsequently queued workflow run to interrupt and cancel previous runs | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES: "airflow.* astro.* astro_databricks.*" | |
| CI_ENABLED: true | |
| jobs: | |
| Markdown-Link-Check: | |
| if: github.event.action != 'labeled' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: gaurav-nelson/github-action-markdown-link-check@v1 | |
| with: | |
| config-file: '.github/workflows/mlc_config.json' | |
| # TODO: Fix Type-check failures ticket: https://github.com/astronomer/astro-provider-databricks/issues/5 | |
| # Type-Check: | |
| # if: github.event.action != 'labeled' | |
| # runs-on: ubuntu-latest | |
| # env: | |
| # MYPY_FORCE_COLOR: 1 | |
| # TERM: xterm-color | |
| # SETUPTOOLS_USE_DISTUTILS: stdlib | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # - uses: actions/setup-python@v3 | |
| # with: | |
| # python-version: '3.8' | |
| # architecture: 'x64' | |
| # - uses: actions/cache@v3 | |
| # with: | |
| # path: | | |
| # ~/.cache/pip | |
| # .nox | |
| # key: ${{ runner.os }}-${{ hashFiles('pyproject.toml') }} | |
| # - run: pip3 install nox packaging | |
| # - run: nox -s type_check | |
| Build-Docs: | |
| if: github.event.action != 'labeled' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: '3.9' | |
| architecture: 'x64' | |
| - uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| .nox | |
| key: ${{ runner.os }}-${{ hashFiles('pyproject.toml') }} | |
| - run: pip3 install nox packaging | |
| - run: nox -s build_docs | |
| Run-Unit-Tests: # yamllint disable-line | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: ['3.8', '3.9', '3.10'] | |
| airflow: [2.7] | |
| if: >- | |
| github.event_name == 'push' || | |
| ( | |
| github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.fork == false | |
| ) || | |
| ( | |
| github.event_name == 'pull_request_target' && | |
| contains(github.event.pull_request.labels.*.name, 'safe to test') | |
| ) || | |
| ( | |
| github.event_name == 'release' | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| if: github.event_name != 'pull_request_target' | |
| - name: Checkout pull/${{ github.event.number }} | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| if: github.event_name == 'pull_request_target' | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| architecture: 'x64' | |
| - uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| .nox | |
| key: unit-tests-os-${{ runner.os }}-python-${{ matrix.python }}-airflow-${{ matrix.airflow }}-deps-${{ hashFiles('pyproject.toml') }}-version-${{ hashFiles('src/astro_databricks/__init__.py') }} | |
| - run: pip3 install nox packaging | |
| - run: nox -s "test-${{ matrix.python }}(airflow='${{ matrix.airflow }}')" -- --ignore "tests/test_example_dags.py" --cov-report=xml | |
| - name: Upload coverage | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: coverage-unit-tests-${{ matrix.python }}-${{ matrix.airflow }} | |
| path: ./.coverage | |
| Run-Example-DAGs: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: ['3.8', '3.9', '3.10'] | |
| airflow: ['2.7', '2.8'] | |
| if: >- | |
| github.event_name == 'push' || | |
| ( | |
| github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.fork == false | |
| ) || | |
| ( | |
| github.event_name == 'pull_request_target' && | |
| contains(github.event.pull_request.labels.*.name, 'safe to test') | |
| )|| | |
| ( | |
| github.event_name == 'release' | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| if: github.event_name != 'pull_request_target' | |
| - name: Checkout pull/${{ github.event.number }} | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| if: github.event_name == 'pull_request_target' | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| architecture: 'x64' | |
| - uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| .nox | |
| key: example-dags-os-${{ runner.os }}-python-${{ matrix.python }}-airflow-${{ matrix.airflow }}-deps-${{ hashFiles('pyproject.toml') }}-nox-${{ hashFiles('noxfile.py') }}-version-${{ hashFiles('src/astro_databricks/__init__.py') }} | |
| - run: cat .github/ci-test-connections.yaml > test-connections.yaml | |
| - run: sqlite3 /tmp/sqlite_default.db "VACUUM;" | |
| - run: pip3 install nox packaging | |
| - run: DATABRICKS_GROUP_ID="${{github.run_id}}_${{matrix.python}}_${{matrix.airflow}}" nox -s "test-${{ matrix.python }}(airflow='${{ matrix.airflow }}')" -- "tests/test_example_dags.py" --cov-report=xml | |
| - name: Upload coverage | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: coverage-example-dags-${{ matrix.python }}-${{ matrix.airflow }} | |
| path: ./.coverage | |
| env: | |
| DATABRICKS_CONN_TOKEN: ${{ secrets.DATABRICKS_CONN_TOKEN }} | |
| DATABRICKS_CONN_HOST: ${{ secrets.DATABRICKS_CONN_HOST }} | |
| DATABRICKS_CONN: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} | |
| Code-Coverage: | |
| if: github.event.action != 'labeled' | |
| needs: | |
| - Run-Unit-Tests | |
| - Run-Example-DAGs | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: 3.8 | |
| - name: Install coverage | |
| run: | | |
| pip3 install coverage | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| path: ./coverage | |
| - name: Run coverage | |
| run: | | |
| coverage combine ./coverage/coverage*/.coverage | |
| coverage report | |
| coverage xml | |
| - name: Upload coverage | |
| uses: codecov/codecov-action@v2 | |
| with: | |
| fail_ci_if_error: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ./coverage.xml | |
| Publish-Package: | |
| if: github.event_name == 'release' | |
| name: Build and publish Python 🐍 distributions 📦 to PyPI | |
| needs: | |
| - Markdown-Link-Check | |
| - Build-Docs | |
| - Run-Unit-Tests | |
| - Run-Example-DAGs | |
| - Code-Coverage | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.8' | |
| architecture: 'x64' | |
| - run: pip3 install hatch | |
| - run: hatch build | |
| - run: hatch publish | |
| env: | |
| HATCH_INDEX_USER: __token__ | |
| HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }} |