From 29c3a587fdbab24c17875e253bc6ff2018510c59 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Mon, 9 Dec 2024 09:30:26 +0000 Subject: [PATCH 1/4] Remove 'dbt-core<1.8.9' pin Closes: #1343 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1ae97ee6d..ba56bde19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,7 +144,7 @@ dependencies = [ "types-requests", "types-python-dateutil", "Werkzeug<3.0.0", - "dbt-core<1.8.9" # TODO: https://github.com/astronomer/astronomer-cosmos/issues/1343 + "dbt-core" ] pre-install-commands = ["sh scripts/test/pre-install-airflow.sh {matrix:airflow} {matrix:python}"] From 40568e1014c928a4020c708c8d4efaab8cc5eabb Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Mon, 9 Dec 2024 15:39:00 +0000 Subject: [PATCH 2/4] Try to solve issue 1343 - it seems a caching issue --- .github/workflows/test.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a80a62d60..4c7950042 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: push: # Run on pushes to the default branch - branches: [main] + branches: [main, issue-1343] pull_request_target: # Also run on pull requests originated from forks branches: [main] @@ -134,12 +134,12 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.ref }} - - uses: actions/cache@v3 - with: - path: | - ~/.cache/pip - .local/share/hatch/ - key: integration-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + #- uses: actions/cache@v3 + # with: + # path: | + # ~/.cache/pip + # .local/share/hatch/ + # key: integration-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -211,12 +211,12 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha || github.ref }} - - uses: actions/cache@v3 - with: - path: | - ~/.cache/pip - .local/share/hatch/ - key: integration-expensive-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + #- uses: actions/cache@v3 + # with: + # path: | + # ~/.cache/pip + # .local/share/hatch/ + # key: integration-expensive-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 From 8b052bde18da47d26769b4c1786e07177ecb5bd2 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Mon, 9 Dec 2024 15:48:23 +0000 Subject: [PATCH 3/4] Attempt to solve databricks dependency issue --- scripts/test/integration-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test/integration-setup.sh b/scripts/test/integration-setup.sh index fec9e95eb..7eca073c4 100644 --- a/scripts/test/integration-setup.sh +++ b/scripts/test/integration-setup.sh @@ -6,9 +6,9 @@ set -e # we install using the following workaround to overcome installation conflicts, such as: # apache-airflow 2.3.0 and dbt-core [0.13.0 - 1.5.2] and jinja2>=3.0.0 because these package versions have conflicting dependencies -pip uninstall -y dbt-postgres dbt-databricks dbt-vertica +pip uninstall -y 'dbt-bigquery' 'dbt-databricks' 'dbt-postgres' 'dbt-vertica' 'dbt-core' rm -rf airflow.* pip freeze | grep airflow airflow db reset -y airflow db init -pip install 'dbt-core' 'dbt-bigquery' 'dbt-databricks' 'dbt-postgres' 'dbt-vertica' 'openlineage-airflow' +pip install 'dbt-databricks' 'dbt-bigquery' 'dbt-postgres' 'dbt-vertica' 'openlineage-airflow' From 7c18bf4c2b748b79a66e035fc538b7ec933b6dec Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Mon, 9 Dec 2024 16:53:24 +0000 Subject: [PATCH 4/4] Tidy up PR, remove unnecessary changes --- .github/workflows/test.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c7950042..a80a62d60 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: push: # Run on pushes to the default branch - branches: [main, issue-1343] + branches: [main] pull_request_target: # Also run on pull requests originated from forks branches: [main] @@ -134,12 +134,12 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.ref }} - #- uses: actions/cache@v3 - # with: - # path: | - # ~/.cache/pip - # .local/share/hatch/ - # key: integration-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .local/share/hatch/ + key: integration-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -211,12 +211,12 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha || github.ref }} - #- uses: actions/cache@v3 - # with: - # path: | - # ~/.cache/pip - # .local/share/hatch/ - # key: integration-expensive-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .local/share/hatch/ + key: integration-expensive-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4