Merge branch 'main' into make-gcp-dataset-profile-arg-optional

astronomer · May 29, 2024 · ef78fe9 · ef78fe9
2 parents 0dfcd9d + 32c47ca
commit ef78fe9
Show file tree

Hide file tree

Showing 63 changed files with 1,505 additions and 248 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1 +1 @@
-* @astronomer/astro-cosmos-admins @astronomer/astro-devex
+@astronomer/astro-cosmos-admins @astronomer/astro-cosmos-codeowners
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install build dependencies
         run: python -m pip install --upgrade build
@@ -37,7 +37,7 @@ jobs:
           path: dist
 
       - name: Push build artifacts to PyPi
-        uses: pypa/gh-action-pypi-publish@v1.6.4
+        uses: pypa/gh-action-pypi-publish@v1.8.14
         with:
           user: __token__
           password: ${{ secrets.PYPI_TOKEN }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -39,10 +39,8 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11"]
-        airflow-version: ["2.3", "2.4", "2.5", "2.6", "2.7", "2.8"]
+        airflow-version: ["2.4", "2.5", "2.6", "2.7", "2.8", "2.9"]
         exclude:
-          - python-version: "3.11"
-            airflow-version: "2.3"
           - python-version: "3.11"
             airflow-version: "2.4"
     steps:
@@ -83,10 +81,8 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11"]
-        airflow-version: ["2.3", "2.4", "2.5", "2.6", "2.7", "2.8"]
+        airflow-version: ["2.4", "2.5", "2.6", "2.7", "2.8", "2.9"]
         exclude:
-          - python-version: "3.11"
-            airflow-version: "2.3"
           - python-version: "3.11"
             airflow-version: "2.4"
     services:
@@ -125,17 +121,16 @@ jobs:
       - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }}
         run: |
           hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup
-          DATABRICKS_UNIQUE_ID="${{github.run_id}}_${{matrix.python-version}}_${{ matrix.airflow-version }}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration
+          hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration
         env:
           AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
           AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres
-          AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }}
+          DATABRICKS_HOST: mock
+          DATABRICKS_WAREHOUSE_ID: mock
+          DATABRICKS_TOKEN: mock
+          DATABRICKS_CLUSTER_ID: mock
           AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
           PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
-          DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
-          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
-          DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }}
-          DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
           COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }}
           POSTGRES_HOST: localhost
           POSTGRES_USER: postgres
@@ -201,11 +196,8 @@ jobs:
           AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres
           PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
           AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }}
-          AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
-          DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
-          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
-          DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }}
           DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
+          AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
           COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }}
           POSTGRES_HOST: localhost
           POSTGRES_USER: postgres
@@ -225,9 +217,6 @@ jobs:
       AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres
       PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
       AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }}
-      DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
-      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
-      DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }}
       DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
 
   Run-Integration-Tests-Sqlite:
@@ -268,12 +257,11 @@ jobs:
           AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres
           AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
           PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
-          AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }}
-          DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
-          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
-          DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }}
-          DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
           COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }}
+          DATABRICKS_CLUSTER_ID: mock
+          DATABRICKS_HOST: mock
+          DATABRICKS_WAREHOUSE_ID: mock
+          DATABRICKS_TOKEN: mock
           POSTGRES_HOST: localhost
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -292,6 +280,78 @@ jobs:
       AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres
       PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
 
+  Run-Integration-Tests-DBT-1-5-4:
+    needs: Authorize
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.11" ]
+        airflow-version: [ "2.7" ]
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pip
+            .nox
+          key: integration-dbt-1-5-4-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install packages and dependencies
+        run: |
+          python -m pip install hatch
+          hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze
+
+      - name: Test Cosmos against Airflow ${{ matrix.airflow-version }}, Python ${{ matrix.python-version }} and dbt 1.5.4
+        run: |
+          hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-dbt-1-5-4
+        env:
+          AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
+          AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres
+          AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
+          PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
+          COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }}
+          DATABRICKS_CLUSTER_ID: mock
+          DATABRICKS_HOST: mock
+          DATABRICKS_WAREHOUSE_ID: mock
+          DATABRICKS_TOKEN: mock
+          POSTGRES_HOST: localhost
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+          POSTGRES_SCHEMA: public
+          POSTGRES_PORT: 5432
+
+      - name: Upload coverage to Github
+        uses: actions/upload-artifact@v2
+        with:
+          name: coverage-integration-dbt-1-5-4-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}
+          path: .coverage
+
+    env:
+      AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
+      AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres
+      PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
+
   Run-Performance-Tests:
     needs: Authorize
     runs-on: ubuntu-latest

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -10,7 +10,7 @@ repos:
         types: [file]
         pass_filenames: false
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-merge-conflict
@@ -54,13 +54,13 @@ repos:
           - --py37-plus
           - --keep-runtime-typing
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.4
+    rev: v0.4.4
     hooks:
       - id: ruff
         args:
           - --fix
   - repo: https://github.com/psf/black
-    rev: 24.3.0
+    rev: 24.4.2
     hooks:
       - id: black
         args: ["--config", "./pyproject.toml"]
@@ -71,7 +71,7 @@ repos:
         alias: black
         additional_dependencies: [black>=22.10.0]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.9.0"
+    rev: "v1.10.0"
 
     hooks:
       - id: mypy

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,18 +1,38 @@
 Changelog
 =========
 
-1.4.0a1 (2024-02-29)
+1.4.1 (2024-05-17)
+--------------------
+
+Bug fixes
+
+* Fix manifest testing behavior in #955 by @chris-okorodudu
+* Handle ValueError when unpacking partial_parse.msgpack in #972 by @tatiana
+
+Others
+
+* Enable pre-commit run and fix type-check job by @pankajastro in #957
+* Clean databricks credentials in test/CI in #969 by @tatiana
+* Update CODEOWNERS in #969 by @tatiana
+* Update emeritus contributors list in #961 by @tatiana
+* Promote @dwreeves to committer in #960 by @tatiana
+* Pre-commit hook updates in #956
+
+
+1.4.0 (2024-05-13)
 --------------------
 
 Features
 
 * Add dbt docs natively in Airflow via plugin by @dwreeves in #737
 * Add support for ``InvocationMode.DBT_RUNNER`` for local execution mode by @jbandoro in #850
 * Support partial parsing to render DAGs faster when using ``ExecutionMode.LOCAL``, ``ExecutionMode.VIRTUALENV`` and ``LoadMode.DBT_LS`` by @dwreeves in #800
+* Improve performance by 22-35% or more by caching partial parse artefact by @tatiana in #904
 * Add Azure Container Instance as Execution Mode by @danielvdende in #771
 * Add dbt build operators by @dylanharper-qz in #795
 * Add dbt profile config variables to mapped profile by @ykuc in #794
 * Add more template fields to ``DbtBaseOperator`` by @dwreeves in #786
+* Add ``pip_install_options`` argument to operators by @octiva in #808
 
 Bug fixes
 
@@ -21,24 +41,48 @@ Bug fixes
 * Improve ``dbt ls`` parsing resilience to missing tags/config by @tatiana in #859
 * Fix ``operator_args`` modified in place in Airflow converter by @jbandoro in #835
 * Fix Docker and Kubernetes operators execute method resolution by @jbandoro in #849
+* Fix ``TrinoBaseProfileMapping`` required parameter for non method authentication by @AlexandrKhabarov in #921
+* Fix global flags for lists by @ms32035 in #863
+* Fix ``GoogleCloudServiceAccountDictProfileMapping`` when getting values from the Airflow connection ``extra__`` keys by @glebkrapivin in #923
+* Fix using the dag as a keyword argument as ``specific_args_keys`` in DbtTaskGroup by @tboutaour in #916
+* Fix ACI integration (``DbtAzureContainerInstanceBaseOperator``) by @danielvdende in #872
+* Fix setting dbt project dir to the tmp dir by @dwreeves in #873
+* Fix dbt docs operator to not use ``graph.gpickle`` file when ``--no-write-json`` is passed by @dwreeves in #883
+* Make Pydantic a required dependency by @pankajkoti in #939
+* Gracefully error if users try to ``emit_datasets`` with ``Airflow 2.9.0`` or ``2.9.1`` by @tatiana in #948
+* Fix parsing tests that have no parents in #933 by @jlaneve
+* Correct ``root_path`` in partial parse cache by @pankajkoti in #950
 
 Docs
 
 * Fix docs homepage link by @jlaneve in #860
 * Fix docs ``ExecutionConfig.dbt_project_path`` by @jbandoro in #847
 * Fix typo in MWAA getting started guide by @jlaneve in #846
+* Fix typo related to exporting docs to GCS by @tboutaour in #922
+* Improve partial parsing docs by @tatiana in #898
+* Improve docs for datasets for airflow >= 2.4 by @SiddiqueAhmad in #879
+* Improve test behaviour docs to highlight ``warning`` feature in the ``virtualenv`` mode by @mc51 in #910
+* Fix docs typo by @SiddiqueAhmad in #917
+* Improve Astro docs by @RNHTTR in #951
 
 Others
 
 * Add performance integration tests by @jlaneve in #827
+* Enable ``append_env`` in ``operator_args`` by default by @tatiana in #899
+* Change default ``append_env`` behaviour depending on Cosmos ``ExecutionMode`` by @pankajkoti and @pankajastro in #954
+* Expose the ``dbt`` graph in the ``DbtToAirflowConverter`` class by @tommyjxl in #886
+* Improve dbt docs plugin rendering padding by @dwreeves in #876
 * Add ``connect_retries`` to databricks profile to fix expensive integration failures by @jbandoro in #826
 * Add import sorting (isort) to Cosmos by @jbandoro in #866
 * Add Python 3.11 to CI/tests by @tatiana and @jbandoro in #821, #824 and #825
 * Fix failing ``test_created_pod`` for ``apache-airflow-providers-cncf-kubernetes`` after v8.0.0 update by @jbandoro in #854
 * Extend ``DatabricksTokenProfileMapping`` test to include session properties by @tatiana in #858
 * Fix broken integration test uncovered from Pytest 8.0 update by @jbandoro in #845
-* Pre-commit hook updates in #834, #843 and #852
-
+* Add Apache Airflow 2.9 to the test matrix by @tatiana in #940
+* Replace deprecated ``DummyOperator`` by ``EmptyOperator`` if Airflow >=2.4.0 by @tatiana in #900
+* Improve logs to troubleshoot issue in 1.4.0a2 with astro-cli by @tatiana in #947
+* Fix issue when publishing a new release to PyPI by @tatiana in #946
+* Pre-commit hook updates in #820, #834, #843 and #852, #890, #896, #901, #905, #908, #919, #931, #941
 
 
 1.3.2 (2024-01-26)

diff --git a/CODEOWNERS b/CODEOWNERS
@@ -0,0 +1 @@
+@astronomer/astro-cosmos-admins @astronomer/astro-cosmos-codeowners @jbandoro @dwreeves
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
@@ -5,7 +5,7 @@
 We as members, contributors, and leaders pledge to make participation in our
 community a harassment-free experience for everyone, regardless of age, body
 size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
+identity and expression, level of experience, education, socioeconomic status,
 nationality, personal appearance, race, religion, or sexual identity
 and orientation.
 

diff --git a/cosmos/__init__.py b/cosmos/__init__.py
@@ -5,7 +5,7 @@
 
 Contains dags, task groups, and operators.
 """
-__version__ = "1.4.0a1"
+__version__ = "1.4.1"
 
 
 from cosmos.airflow.dag import DbtDag
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		* @astronomer/astro-cosmos-admins @astronomer/astro-devex
		@astronomer/astro-cosmos-admins @astronomer/astro-cosmos-codeowners
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		@astronomer/astro-cosmos-admins @astronomer/astro-cosmos-codeowners @jbandoro @dwreeves