From 85c330201db8de0f5073bfd367a9d4b545e3ce7d Mon Sep 17 00:00:00 2001
From: huangxingyi <99020960+huangxingyi-git@users.noreply.github.com>
Date: Tue, 26 Nov 2024 02:28:43 +0900
Subject: [PATCH 1/4] Set unique temp table suffix to allow parallel merge
 execution (#854)

Signed-off-by: huangxingyi <hxy911122@gmail.com>
Co-authored-by: Ben Cassell <98852248+benc-db@users.noreply.github.com>
---
 CHANGELOG.md                                                    | 1 +
 .../macros/materializations/incremental/incremental.sql         | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2a58b1c9..e587078d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ### Features
 
+- Add config for generating unique tmp table names for enabling parralel merge (thanks @huangxingyi-git!) ([854](https://github.com/databricks/dbt-databricks/pull/854))
 - Add support for serverless job clusters on python models ([706](https://github.com/databricks/dbt-databricks/pull/706))
 - Add 'user_folder_for_python' behavior to switch writing python model notebooks to the user's folder ([835](https://github.com/databricks/dbt-databricks/pull/835))
 - Merge capabilities are extended ([739](https://github.com/databricks/dbt-databricks/pull/739)) to include the support for the following features (thanks @mi-volodin):
diff --git a/dbt/include/databricks/macros/materializations/incremental/incremental.sql b/dbt/include/databricks/macros/materializations/incremental/incremental.sql
index 6096be4a..55c54324 100644
--- a/dbt/include/databricks/macros/materializations/incremental/incremental.sql
+++ b/dbt/include/databricks/macros/materializations/incremental/incremental.sql
@@ -19,7 +19,7 @@
   {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}
   {%- set target_relation = this.incorporate(type='table') -%}
   {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier, needs_information=True) -%}
-  {%- if unique_tmp_table_suffix == True and raw_strategy == 'replace_where' and raw_file_format == 'delta' -%}
+  {%- if unique_tmp_table_suffix == True and raw_strategy in ['merge', 'replace_where'] and raw_file_format == 'delta' -%}
     {%- set temp_relation_suffix = adapter.generate_unique_temporary_table_suffix() -%}
   {%- else -%}
     {%- set temp_relation_suffix = '__dbt_tmp' -%}

From 97d6381d04d2b8fb830493bd9e4a9c5cc05e69dc Mon Sep 17 00:00:00 2001
From: Ben Cassell <98852248+benc-db@users.noreply.github.com>
Date: Mon, 25 Nov 2024 15:01:17 -0800
Subject: [PATCH 2/4] Try setting queue enabled for Python notebook jobs (#856)

---
 CHANGELOG.md                                                | 1 +
 dbt/adapters/databricks/python_models/python_submissions.py | 1 +
 tests/unit/python/test_python_job_support.py                | 2 ++
 3 files changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e587078d..a3a31a6d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,7 @@
 ### Fixes
 
 - Replace array indexing with 'get' in split_part so as not to raise exception when indexing beyond bounds ([839](https://github.com/databricks/dbt-databricks/pull/839))
+- Set queue enabled for Python notebook jobs ([856](https://github.com/databricks/dbt-databricks/pull/856))
 
 ### Under the Hood
 
diff --git a/dbt/adapters/databricks/python_models/python_submissions.py b/dbt/adapters/databricks/python_models/python_submissions.py
index 27e4303c..5f0f7653 100644
--- a/dbt/adapters/databricks/python_models/python_submissions.py
+++ b/dbt/adapters/databricks/python_models/python_submissions.py
@@ -224,6 +224,7 @@ def compile(self, path: str) -> PythonJobDetails:
         if access_control_list:
             job_spec["access_control_list"] = access_control_list
 
+        job_spec["queue"] = {"enabled": True}
         return PythonJobDetails(self.run_name, job_spec, additional_job_config)
 
 
diff --git a/tests/unit/python/test_python_job_support.py b/tests/unit/python/test_python_job_support.py
index 4eec7355..41f48041 100644
--- a/tests/unit/python/test_python_job_support.py
+++ b/tests/unit/python/test_python_job_support.py
@@ -158,6 +158,7 @@ def test_compile__empty_configs(self, client, permission_builder, parsed_model,
                 "notebook_path": "path",
             },
             "libraries": [],
+            "queue": {"enabled": True},
         }
         assert details.additional_job_config == {}
 
@@ -182,5 +183,6 @@ def test_compile__nonempty_configs(self, client, permission_builder, parsed_mode
             "cluster_id": "id",
             "libraries": [{"pypi": {"package": "foo"}}],
             "access_control_list": [{"user_name": "user", "permission_level": "IS_OWNER"}],
+            "queue": {"enabled": True},
         }
         assert details.additional_job_config == {"foo": "bar"}

From b95a04a59619fc1ee168a6a836e824d030ea0d8a Mon Sep 17 00:00:00 2001
From: Ben Cassell <98852248+benc-db@users.noreply.github.com>
Date: Tue, 26 Nov 2024 08:43:45 -0800
Subject: [PATCH 3/4] Migrating to pyproject.toml and Hatch (#853)

Co-authored-by: Alex Guo <133057192+alexguo-db@users.noreply.github.com>
---
 .github/workflows/integration.yml             |  60 ++++---
 .github/workflows/main.yml                    | 150 +++--------------
 .gitignore                                    |   1 +
 .pre-commit-config.yaml                       |  15 ++
 CHANGELOG.md                                  |   1 +
 MANIFEST.in                                   |   1 -
 dbt/adapters/databricks/__version__.py        |   2 +-
 dbt/adapters/databricks/impl.py               |   4 +-
 dbt/adapters/databricks/logging.py            |   8 +-
 .../python_models/python_submissions.py       |   9 +-
 dev-requirements.txt                          |  15 --
 docs/dbt-databricks-dev.md                    | 153 ++++++++++++++++++
 mypy.ini                                      |  19 ---
 pyproject.toml                                | 131 +++++++++++++++
 pytest.ini                                    |  14 --
 requirements.txt                              |   9 --
 ruff.toml                                     |   6 -
 scripts/build-dist.sh                         |  20 ---
 setup.py                                      |  81 ----------
 .../materialized_view_tests/fixtures.py       |   8 +-
 .../adapter/streaming_tables/fixtures.py      |   8 +-
 tests/unit/api_client/api_test_base.py        |   2 +-
 tests/unit/api_client/test_cluster_api.py     |   3 +-
 tests/unit/api_client/test_command_api.py     |   3 +-
 .../api_client/test_command_context_api.py    |   3 +-
 tests/unit/api_client/test_job_runs_api.py    |   3 +-
 tests/unit/api_client/test_workspace_api.py   |   2 +-
 tests/unit/events/test_connection_events.py   |   2 +-
 tests/unit/events/test_cursor_events.py       |   2 +-
 .../macros/adapters/test_python_macros.py     |   3 +-
 tests/unit/macros/base.py                     |   2 +-
 .../unit/macros/relations/test_view_macros.py |   3 +-
 tests/unit/python/test_python_run_tracker.py  |   2 +-
 tests/unit/python/test_python_submitters.py   |   4 +-
 tests/unit/relation_configs/test_comment.py   |   3 +-
 .../test_materialized_view_config.py          |   3 +-
 .../relation_configs/test_partitioning.py     |   2 +-
 tests/unit/relation_configs/test_query.py     |   3 +-
 tests/unit/relation_configs/test_refresh.py   |   3 +-
 .../test_streaming_table_config.py            |   2 +-
 tests/unit/relation_configs/test_tags.py      |   3 +-
 .../relation_configs/test_tblproperties.py    |   3 +-
 tests/unit/test_adapter.py                    |  43 +++--
 tests/unit/test_compute_config.py             |   3 +-
 tox.ini                                       |  71 --------
 45 files changed, 438 insertions(+), 450 deletions(-)
 create mode 100644 .pre-commit-config.yaml
 delete mode 100644 MANIFEST.in
 delete mode 100644 dev-requirements.txt
 create mode 100644 docs/dbt-databricks-dev.md
 delete mode 100644 mypy.ini
 create mode 100644 pyproject.toml
 delete mode 100644 pytest.ini
 delete mode 100644 requirements.txt
 delete mode 100644 ruff.toml
 delete mode 100755 scripts/build-dist.sh
 delete mode 100644 setup.py
 delete mode 100644 tox.ini

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 23ef5275..b3503372 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run-tox-tests-uc-cluster:
+  run-uc-cluster-e2e-tests:
     runs-on: ubuntu-latest
     environment: azure-prod
     env:
@@ -26,22 +26,26 @@ jobs:
       TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }}
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
       - name: Set up python
         id: setup-python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.9"
+
       - name: Get http path from environment
         run: python .github/workflows/build_cluster_http_path.py
         shell: sh
-      - name: Install tox
+
+      - name: Install Hatch
         id: install-dependencies
-        run: pip install tox
-      - name: Run integration-uc-databricks-cluster
-        run: DBT_TEST_USER=notnecessaryformosttests@example.com DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH=$DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET tox -e integration-databricks-uc-cluster
+        uses: pypa/hatch@install
 
-  run-tox-tests-uc-sql:
+      - name: Run UC Cluster Functional Tests
+        run: DBT_TEST_USER=notnecessaryformosttests@example.com DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH=$DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET hatch -v run uc-cluster-e2e
+
+  run-sqlwarehouse-e2e-tests:
     runs-on: ubuntu-latest
     environment: azure-prod
     env:
@@ -54,22 +58,26 @@ jobs:
       TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }}
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
       - name: Set up python
         id: setup-python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.9"
+
       - name: Get http path from environment
         run: python .github/workflows/build_cluster_http_path.py
         shell: sh
-      - name: Install tox
+
+      - name: Install Hatch
         id: install-dependencies
-        run: pip install tox
-      - name: Run integration-databricks-uc-sql-endpoint
-        run: DBT_TEST_USER=notnecessaryformosttests@example.com DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH=$DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET tox -e integration-databricks-uc-sql-endpoint
+        uses: pypa/hatch@install
+
+      - name: Run Sql Endpoint Functional Tests
+        run: DBT_TEST_USER=notnecessaryformosttests@example.com DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH=$DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET hatch -v run sqlw-e2e
 
-  run-tox-tests-non-uc:
+  run-cluster-e2e-tests:
     runs-on: ubuntu-latest
     environment: azure-prod
     env:
@@ -79,17 +87,21 @@ jobs:
       DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
       - name: Set up python
         id: setup-python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.9"
+
       - name: Get http path from environment
         run: python .github/workflows/build_cluster_http_path.py
         shell: sh
-      - name: Install tox
+
+      - name: Install Hatch
         id: install-dependencies
-        run: pip install tox
-      - name: Run integration-databricks-cluster
-        run: DBT_TEST_USER=notnecessaryformosttests@example.com DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_HTTP_PATH=$DBT_DATABRICKS_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET tox -e integration-databricks-cluster
+        uses: pypa/hatch@install
+
+      - name: Run Cluster Functional Tests
+        run: DBT_TEST_USER=notnecessaryformosttests@example.com DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_HTTP_PATH=$DBT_DATABRICKS_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET hatch -v run cluster-e2e
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3a1637c8..1b9a99cd 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -42,40 +42,28 @@ defaults:
 
 jobs:
   code-quality:
-    name: ${{ matrix.toxenv }}
+    name: Code Quality
 
     runs-on: ubuntu-latest
     timeout-minutes: 10
 
     strategy:
       fail-fast: false
-      matrix:
-        toxenv: [linter]
-
-    env:
-      TOXENV: ${{ matrix.toxenv }}
-      PYTEST_ADDOPTS: "-v --color=yes"
 
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v3
-        with:
-          persist-credentials: false
+        uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4.3.0
+        uses: actions/setup-python@v5
         with:
           python-version: "3.9"
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip --version
-          python -m pip install tox
-          tox --version
+      - name: Install Hatch
+        uses: pypa/hatch@install
 
-      - name: Run tox
-        run: tox
+      - name: Run Code Quality
+        run: hatch -v run code-quality
 
   unit:
     name: unit test / python ${{ matrix.python-version }}
@@ -88,131 +76,39 @@ jobs:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
 
-    env:
-      TOXENV: "unit"
-      PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"
-
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v3
-        with:
-          persist-credentials: false
+        uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4.3.0
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip --version
-          python -m pip install tox
-          tox --version
-      - name: Run tox
-        run: tox
-
-      - name: Get current date
-        if: always()
-        id: date
-        run: echo "::set-output name=date::$(date +'%Y-%m-%dT%H_%M_%S')" #no colons allowed for artifacts
-
-      - uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
-          path: unit_results.csv
+      - name: Install Hatch
+        uses: pypa/hatch@install
 
-  build:
-    name: build packages
+      - name: Run Unit Tests
+        run: hatch run -v +py=${{ matrix.python-version }} test:unit
 
+  build:
+    name: Build and Verify Packages
     runs-on: ubuntu-latest
 
-    outputs:
-      is_alpha: ${{ steps.check-is-alpha.outputs.is_alpha }}
-
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4.3.0
+        uses: actions/setup-python@v5
         with:
           python-version: "3.9"
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip install --upgrade setuptools wheel twine check-wheel-contents
-          python -m pip --version
-      - name: Build distributions
-        run: ./scripts/build-dist.sh
-
-      - name: Show distributions
-        run: ls -lh dist/
-
-      - name: Check distribution descriptions
-        run: |
-          twine check dist/*
-      - name: Check wheel contents
-        run: |
-          check-wheel-contents dist/*.whl --ignore W007,W008
-
-      - name: Check if this is an alpha version
-        id: check-is-alpha
-        run: |
-          export is_alpha=0
-          if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi
-          echo "::set-output name=is_alpha::$is_alpha"
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: dist
-          path: dist/
-
-  test-build:
-    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }}
-
-    if: needs.build.outputs.is_alpha == 0
+      - name: Install Hatch
+        uses: pypa/hatch@install
 
-    needs: build
-
-    runs-on: ${{ matrix.os }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-
-    steps:
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4.3.0
-        with:
-          python-version: ${{ matrix.python-version }}
+      - name: Build distributions
+        run: hatch -v build
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install --user --upgrade pip
-          python -m pip install --upgrade wheel
-          python -m pip --version
-      - uses: actions/download-artifact@v3
-        with:
-          name: dist
-          path: dist/
-
-      - name: Show distributions
-        run: ls -lh dist/
-
-      - name: Install wheel distributions
-        run: |
-          find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-      - name: Check wheel distributions
-        run: |
-          dbt --version
-      - name: Install source distributions
-        run: |
-          find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-      - name: Check source distributions
-        run: |
-          dbt --version
+      - name: Verify distributions
+        run: hatch run verify:check-all
diff --git a/.gitignore b/.gitignore
index 3d42d669..0725a26b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,4 @@ logs/
 .venv*
 *.sublime*
 .python-version
+.hatch
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 00000000..81ca0ce9
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,15 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.8.0
+    hooks:
+      # Run the linter.
+      - id: ruff
+      # Run the formatter.
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.13.0
+    hooks:
+      - id: mypy
+        additional_dependencies: [types-requests]
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a3a31a6d..dfce1d4e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -34,6 +34,7 @@
 - Fix behavior flag use in init of DatabricksAdapter (thanks @VersusFacit!) ([836](https://github.com/databricks/dbt-databricks/pull/836))
 - Restrict pydantic to V1 per dbt Labs' request ([843](https://github.com/databricks/dbt-databricks/pull/843))
 - Switching to Ruff for formatting and linting ([847](https://github.com/databricks/dbt-databricks/pull/847))
+- Switching to Hatch and pyproject.toml for project config ([853](https://github.com/databricks/dbt-databricks/pull/853))
 
 ## dbt-databricks 1.8.7 (October 10, 2024)
 
diff --git a/MANIFEST.in b/MANIFEST.in
deleted file mode 100644
index cfbc714e..00000000
--- a/MANIFEST.in
+++ /dev/null
@@ -1 +0,0 @@
-recursive-include dbt/include *.sql *.yml *.md
diff --git a/dbt/adapters/databricks/__version__.py b/dbt/adapters/databricks/__version__.py
index ddfbfc12..a4077fff 100644
--- a/dbt/adapters/databricks/__version__.py
+++ b/dbt/adapters/databricks/__version__.py
@@ -1 +1 @@
-version: str = "1.9.0b1"
+version = "1.9.0b1"
diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py
index ce7edae4..ad1b28da 100644
--- a/dbt/adapters/databricks/impl.py
+++ b/dbt/adapters/databricks/impl.py
@@ -109,7 +109,7 @@
 @dataclass
 class DatabricksConfig(AdapterConfig):
     file_format: str = "delta"
-    table_format: TableFormat = TableFormat.DEFAULT
+    table_format: str = TableFormat.DEFAULT
     location_root: Optional[str] = None
     include_full_name_in_path: bool = False
     partition_by: Optional[Union[list[str], str]] = None
@@ -764,7 +764,7 @@ class RelationAPIBase(ABC, Generic[DatabricksRelationConfig]):
     For the most part, these are just namespaces to group related methods together.
     """
 
-    relation_type: ClassVar[DatabricksRelationType]
+    relation_type: ClassVar[str]
 
     @classmethod
     @abstractmethod
diff --git a/dbt/adapters/databricks/logging.py b/dbt/adapters/databricks/logging.py
index 4b960c36..d0f1d42b 100644
--- a/dbt/adapters/databricks/logging.py
+++ b/dbt/adapters/databricks/logging.py
@@ -1,5 +1,5 @@
-import logging
 import os
+from logging import Handler, LogRecord, getLogger
 from typing import Union
 
 from dbt.adapters.events.logging import AdapterLogger
@@ -7,12 +7,12 @@
 logger = AdapterLogger("Databricks")
 
 
-class DbtCoreHandler(logging.Handler):
+class DbtCoreHandler(Handler):
     def __init__(self, level: Union[str, int], dbt_logger: AdapterLogger):
         super().__init__(level=level)
         self.logger = dbt_logger
 
-    def emit(self, record: logging.LogRecord) -> None:
+    def emit(self, record: LogRecord) -> None:
         # record.levelname will be debug, info, warning, error, or critical
         # these map 1-to-1 with methods of the AdapterLogger
         log_func = getattr(self.logger, record.levelname.lower())
@@ -21,7 +21,7 @@ def emit(self, record: logging.LogRecord) -> None:
 
 dbt_adapter_logger = AdapterLogger("databricks-sql-connector")
 
-pysql_logger = logging.getLogger("databricks.sql")
+pysql_logger = getLogger("databricks.sql")
 pysql_logger_level = os.environ.get("DBT_DATABRICKS_CONNECTOR_LOG_LEVEL", "WARN").upper()
 pysql_logger.setLevel(pysql_logger_level)
 
diff --git a/dbt/adapters/databricks/python_models/python_submissions.py b/dbt/adapters/databricks/python_models/python_submissions.py
index 5f0f7653..cbb27b44 100644
--- a/dbt/adapters/databricks/python_models/python_submissions.py
+++ b/dbt/adapters/databricks/python_models/python_submissions.py
@@ -1,8 +1,8 @@
 from abc import ABC, abstractmethod
 from typing import Any, Optional
 
-from attr import dataclass
 from dbt_common.exceptions import DbtRuntimeError
+from pydantic import BaseModel
 from typing_extensions import override
 
 from dbt.adapters.base import PythonJobHelper
@@ -104,8 +104,7 @@ def upload(self, compiled_code: str) -> str:
         return file_path
 
 
-@dataclass(frozen=True)
-class PythonJobDetails:
+class PythonJobDetails(BaseModel):
     """Details required to submit a Python job run to Databricks."""
 
     run_name: str
@@ -225,7 +224,9 @@ def compile(self, path: str) -> PythonJobDetails:
             job_spec["access_control_list"] = access_control_list
 
         job_spec["queue"] = {"enabled": True}
-        return PythonJobDetails(self.run_name, job_spec, additional_job_config)
+        return PythonJobDetails(
+            run_name=self.run_name, job_spec=job_spec, additional_job_config=additional_job_config
+        )
 
 
 class PythonNotebookSubmitter(PythonSubmitter):
diff --git a/dev-requirements.txt b/dev-requirements.txt
deleted file mode 100644
index 9a1a5911..00000000
--- a/dev-requirements.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-ruff
-freezegun~=1.5.0
-ipdb
-mock>=1.3.0
-mypy==1.1.1
-pytest-csv
-pytest-dotenv
-pytest-xdist
-pytest>=6.0.2
-pytz
-tox>=3.2.0
-types-requests
-types-mock
-
-dbt-tests-adapter>=1.10.2, <2.0
diff --git a/docs/dbt-databricks-dev.md b/docs/dbt-databricks-dev.md
new file mode 100644
index 00000000..1376aac7
--- /dev/null
+++ b/docs/dbt-databricks-dev.md
@@ -0,0 +1,153 @@
+# Preparing to Develop dbt-databricks
+
+## Install Hatch
+
+For best experience, [Hatch](https://hatch.pypa.io/dev/)--a modern Python project manager--should be installed globally.
+Find installation instructions [here](https://hatch.pypa.io/dev/install/#installers).
+
+## Getting Hatch to Work with your IDE
+
+The main thing to getting our project and all expected functionality to work with your IDE is create the default environment for the project, and point your IDE at the interpreter in that project.
+
+First execute
+
+```
+hatch env create
+```
+
+to create the default environment, populating the `.hatch` folder.
+This is where your virtual environments for this project will live.
+In your IDE, you should hopefully see an interpreter in this folder recommended when you enter the set interpreter prompt.
+If not, selecting `.hatch/dbt-databricks/bin/python` as the executable for your interpretor should get you IDE integration.
+If you have an existing `.venv` folder in `dbt-databricks` you should remove it to keep the tools from detecting multiple python virtual environments locally and getting confused.
+
+### VS Code Settings
+
+If you are using VS Code, here are recommended settings (to be included in `.vscode/settings.json`):
+
+```
+{
+    "mypy-type-checker.importStrategy": "fromEnvironment",
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "python.testing.pytestArgs": [
+        "--color=yes",
+        "-n=auto",
+        "--dist=loadscope",
+    ],
+    "[python]": {
+        "editor.insertSpaces": true,
+        "editor.tabSize": 4,
+        "editor.formatOnSave": true,
+        "editor.formatOnType": true,
+        "editor.defaultFormatter": "charliermarsh.ruff",
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": "explicit"
+        },
+    },
+    "logViewer.watch": [
+        {
+            "title": "dbt logs",
+            "pattern": "${workspaceFolder}/logs/**/dbt.log"
+        }
+    ],
+    "logViewer.showStatusBarItemOnChange": true,
+    "editor.formatOnSave": true,
+}
+```
+
+To get all of these features to work, you will need to install 'Log Viewer', 'Mypy Type Checker', and 'Ruff'.
+
+When everything is working as intended you will get the following behaviors:
+
+- [mypy](https://mypy-lang.org/) type-checking
+- [ruff](https://docs.astral.sh/ruff/) formatting and linting (including import organization)
+- [pytest](https://docs.pytest.org/en/stable/) test running from the Test Explorer
+- The ability to quickly jump to test logs as they are produced with the Log Viewer extension
+
+To add test debugging capabilities, add the following to `.vscode/launch.json`:
+
+```
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Debug Tests",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            "purpose": ["debug-test"],
+            "console": "integratedTerminal",
+            "justMyCode": false,
+        }
+    ]
+}
+```
+
+## Hatch from the CLI
+
+There are a number of useful scripts to invoked from the CLI.
+While it is useful to learn about Hatch to understand environments and all of the functionalities, you can run the commands below to accomplish common developer activities:
+
+```
+hatch run setup-precommit
+```
+
+will install our precommit hooks.
+You can run the hooks without installing them with
+
+```
+hatch run code-quality
+```
+
+This runs `ruff` and `mypy` against the project.
+
+```
+hatch run unit
+```
+
+will run the unit tests against Python 3.9, while
+
+```
+hatch run test:unit
+```
+
+will run the unit tests against all supported Python versions.
+
+```
+hatch run {cluster-e2e | uc-cluster-e2e | sqlw-e2e}
+```
+
+will run the functional tests against the HMS cluster, UC cluster, or SQL Warehouse respectively, assuming you have configured your `test.env` file correctly.
+
+```
+hatch build
+```
+
+builds the `sdist` and `wheel` distributables.
+
+If you ever need to install newer versions of a library into the default environment, but don't want to change the dependency version requirements, you can manage this by first entering the shell
+
+```
+hatch shell
+```
+
+and then pip installing as usual, e.g:
+
+```
+pip install dbt-core==1.8.9
+```
+
+## In Case of Emergency
+
+If you are making changes to pyproject.toml, and for whatever reason Hatch isn't respecting your changes, you can blow away existing environments with
+
+```
+hatch env prune
+```
+
+and then recreate the default environment with:
+
+```
+hatch env create
+```
diff --git a/mypy.ini b/mypy.ini
deleted file mode 100644
index 39f04772..00000000
--- a/mypy.ini
+++ /dev/null
@@ -1,19 +0,0 @@
-[mypy]
-strict_optional = True
-no_implicit_optional = True
-disallow_untyped_defs = True
-
-[mypy-tests.*]
-disallow_untyped_defs = False
-
-[mypy-databricks.*]
-ignore_missing_imports = True
-
-[mypy-agate.*]
-ignore_missing_imports = True
-
-[mypy-jinja2.*]
-ignore_missing_imports = True
-
-[mypy-yaml.*]
-ignore_missing_imports = True
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 00000000..f1f680ea
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,131 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dbt-databricks"
+dynamic = ["version"]
+description = "The Databricks adapter plugin for dbt"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.9"
+authors = [{ name = "Databricks", email = "feedback@databricks.com" }]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: POSIX :: Linux",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+dependencies = [
+    "databricks-sdk==0.17.0",
+    "databricks-sql-connector>=3.5.0, <4.0.0",
+    "dbt-adapters>=1.7.0, <2.0",
+    "dbt-common>=1.10.0, <2.0",
+    "dbt-core>=1.8.7, <2.0",
+    "dbt-spark>=1.8.0, <2.0",
+    "keyring>=23.13.0",
+    "pandas<2.2.0",
+    "pydantic>=1.10.0, <2",
+]
+
+[project.urls]
+homepage = "https://github.com/databricks/dbt-databricks"
+changelog = "https://github.com/databricks/dbt-databricks/blob/main/CHANGELOG.md"
+documentation = "https://docs.getdbt.com/reference/resource-configs/databricks-configs"
+issues = "https://github.com/databricks/dbt-databricks/issues"
+repository = "https://github.com/databricks/dbt-databricks"
+
+[tool.hatch.version]
+path = "dbt/adapters/databricks/__version__.py"
+
+[tool.hatch.build]
+include = ["/dbt"]
+
+[tool.hatch.envs.verify]
+detached = true
+dependencies = ["wheel", "twine", "check-wheel-contents"]
+
+[tool.hatch.envs.verify.scripts]
+check-all = ["- check-wheel", "- check-sdist"]
+check-wheel = [
+    "twine check dist/*",
+    "find ./dist/dbt_databricks-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/",
+    "pip freeze | grep dbt-databricks",
+]
+check-sdist = [
+    "check-wheel-contents dist/*.whl --ignore W007,W008",
+    "find ./dist/dbt_databricks-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/",
+    "pip freeze | grep dbt-databricks",
+]
+
+[tool.hatch.envs.default]
+dependencies = [
+    "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git",
+    "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@main",
+    "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git@main#subdirectory=core",
+    "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter",
+    "dbt-spark @ git+https://github.com/dbt-labs/dbt-spark.git@main",
+    "pytest",
+    "pytest-xdist",
+    "pytest-dotenv",
+    "freezegun",
+    "mypy",
+    "pre-commit",
+    "types-requests",
+    "debugpy",
+]
+path = ".hatch"
+python = "3.9"
+
+[tool.hatch.envs.default.scripts]
+setup-precommit = "pre-commit install"
+code-quality = "pre-commit run --all-files"
+unit = "pytest --color=yes -v --profile databricks_cluster -n auto --dist=loadscope tests/unit"
+cluster-e2e = "pytest --color=yes -v --profile databricks_cluster -n auto --dist=loadscope tests/functional"
+uc-cluster-e2e = "pytest --color=yes -v --profile databricks_uc_cluster -n auto --dist=loadscope tests/functional"
+sqlw-e2e = "pytest --color=yes -v --profile databricks_uc_sql_endpoint -n auto --dist=loadscope tests/functional"
+
+[tool.hatch.envs.test.scripts]
+unit = "pytest --color=yes -v --profile databricks_cluster -n auto --dist=loadscope tests/unit"
+
+[[tool.hatch.envs.test.matrix]]
+python = ["3.9", "3.10", "3.11", "3.12"]
+
+[tool.ruff]
+line-length = 100
+target-version = 'py39'
+
+[tool.ruff.lint]
+select = ["E", "W", "F", "I"]
+ignore = ["E203"]
+
+[tool.pytest.ini_options]
+filterwarnings = [
+    "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning",
+    "ignore:unclosed file .*:ResourceWarning",
+]
+env_files = ["test.env"]
+testpaths = ["tests/unit", "tests/functional"]
+markers = [
+    "external: mark test as requiring an external location",
+    "python: mark test as running a python model",
+    "dlt: mark test as running a DLT model",
+]
+
+[tool.mypy]
+strict_optional = true
+no_implicit_optional = true
+disallow_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = "tests.*"
+disallow_untyped_defs = false
+
+[[tool.mypy.overrides]]
+module = ["databricks.*", "agate.*", "jinja2.*", "yaml.*"]
+ignore_missing_imports = true
diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
index 116dc67a..00000000
--- a/pytest.ini
+++ /dev/null
@@ -1,14 +0,0 @@
-[pytest]
-filterwarnings =
-    ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning
-    ignore:unclosed file .*:ResourceWarning
-env_files =
-    test.env
-testpaths =
-    tests/unit
-    tests/integration
-    tests/functional
-markers =
-    external: mark test as requiring an external location
-    python: mark test as running a python model
-    dlt: mark test as running a DLT model
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index bbdfc302..00000000
--- a/requirements.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-databricks-sql-connector>=3.5.0, <4.0
-dbt-spark>=1.8.0, <2.0
-dbt-core>=1.8.7, <2.0
-dbt-common>=1.10.0, <2.0
-dbt-adapters>=1.7.0, <2.0
-databricks-sdk==0.17.0
-keyring>=23.13.0
-protobuf<5.0.0
-pydantic>=1.10.0, <2
diff --git a/ruff.toml b/ruff.toml
deleted file mode 100644
index 3bfb0947..00000000
--- a/ruff.toml
+++ /dev/null
@@ -1,6 +0,0 @@
-line-length = 100
-target-version = 'py39'
-
-[lint]
-select = ["E", "W", "F", "I"]
-ignore = ["E203"]
diff --git a/scripts/build-dist.sh b/scripts/build-dist.sh
deleted file mode 100755
index 3c380839..00000000
--- a/scripts/build-dist.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-DBT_PATH="$( cd "$(dirname "$0")/.." ; pwd -P )"
-
-PYTHON_BIN=${PYTHON_BIN:-python}
-
-echo "$PYTHON_BIN"
-
-set -x
-
-rm -rf "$DBT_PATH"/dist
-rm -rf "$DBT_PATH"/build
-mkdir -p "$DBT_PATH"/dist
-
-cd "$DBT_PATH"
-$PYTHON_BIN setup.py sdist bdist_wheel
-
-set +x
diff --git a/setup.py b/setup.py
deleted file mode 100644
index ea0e6184..00000000
--- a/setup.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-import os
-import sys
-
-# require python 3.9 or newer
-if sys.version_info < (3, 9):
-    print("Error: dbt does not support this version of Python.")
-    print("Please upgrade to Python 3.9 or higher.")
-    sys.exit(1)
-
-
-# require version of setuptools that supports find_namespace_packages
-from setuptools import setup
-
-try:
-    from setuptools import find_namespace_packages
-except ImportError:
-    # the user has a downlevel version of setuptools.
-    print("Error: dbt requires setuptools v40.1.0 or higher.")
-    print('Please upgrade setuptools with "pip install --upgrade setuptools" and try again')
-    sys.exit(1)
-
-
-# pull long description from README
-this_directory = os.path.abspath(os.path.dirname(__file__))
-with open(os.path.join(this_directory, "README.md"), "r", encoding="utf8") as f:
-    long_description = f.read()
-
-
-# get this package's version from dbt/adapters/<name>/__version__.py
-def _get_plugin_version() -> str:
-    _version_path = os.path.join(this_directory, "dbt", "adapters", "databricks", "__version__.py")
-    try:
-        exec(open(_version_path).read())
-        return locals()["version"]
-    except IOError:
-        print("Failed to load dbt-databricks version file for packaging.", file=sys.stderr)
-        sys.exit(-1)
-
-
-package_name = "dbt-databricks"
-package_version = _get_plugin_version()
-description = """The Databricks adapter plugin for dbt"""
-
-setup(
-    name=package_name,
-    version=package_version,
-    description=description,
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    author="Databricks",
-    author_email="feedback@databricks.com",
-    url="https://github.com/databricks/dbt-databricks",
-    packages=find_namespace_packages(include=["dbt", "dbt.*"]),
-    include_package_data=True,
-    install_requires=[
-        "dbt-spark>=1.8.0, <2.0",
-        "dbt-core>=1.8.7, <2.0",
-        "dbt-adapters>=1.7.0, <2.0",
-        "dbt-common>=1.10.0, <2.0",
-        "databricks-sql-connector>=3.5.0, <4.0.0",
-        "databricks-sdk==0.17.0",
-        "keyring>=23.13.0",
-        "pandas<2.2.0",
-        "protobuf<5.0.0",
-        "pydantic>=1.10.0, <2",
-    ],
-    zip_safe=False,
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "License :: OSI Approved :: Apache Software License",
-        "Operating System :: Microsoft :: Windows",
-        "Operating System :: MacOS :: MacOS X",
-        "Operating System :: POSIX :: Linux",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-        "Programming Language :: Python :: 3.12",
-    ],
-    python_requires=">=3.9",
-)
diff --git a/tests/functional/adapter/materialized_view_tests/fixtures.py b/tests/functional/adapter/materialized_view_tests/fixtures.py
index 06a26927..9fa0f041 100644
--- a/tests/functional/adapter/materialized_view_tests/fixtures.py
+++ b/tests/functional/adapter/materialized_view_tests/fixtures.py
@@ -13,9 +13,9 @@ def query_relation_type(project, relation: BaseRelation) -> Optional[str]:
         fetch="one",
     )[0]
     if table_type == "STREAMING_TABLE":
-        return DatabricksRelationType.StreamingTable.value
+        return DatabricksRelationType.StreamingTable
     elif table_type == "MANAGED" or table_type == "EXTERNAL":
-        return DatabricksRelationType.Table.value
+        return DatabricksRelationType.Table
     else:
         is_materialized = project.run_sql(
             "select is_materialized from `system`.`information_schema`.`views`"
@@ -25,9 +25,9 @@ def query_relation_type(project, relation: BaseRelation) -> Optional[str]:
             fetch="one",
         )[0]
         if is_materialized == "TRUE":
-            return DatabricksRelationType.MaterializedView.value
+            return DatabricksRelationType.MaterializedView
         else:
-            return DatabricksRelationType.View.value
+            return DatabricksRelationType.View
 
 
 materialized_view = """
diff --git a/tests/functional/adapter/streaming_tables/fixtures.py b/tests/functional/adapter/streaming_tables/fixtures.py
index 755fbd00..4b9751ee 100644
--- a/tests/functional/adapter/streaming_tables/fixtures.py
+++ b/tests/functional/adapter/streaming_tables/fixtures.py
@@ -13,9 +13,9 @@ def query_relation_type(project, relation: BaseRelation) -> Optional[str]:
         fetch="one",
     )[0]
     if table_type == "STREAMING_TABLE":
-        return DatabricksRelationType.StreamingTable.value
+        return DatabricksRelationType.StreamingTable
     elif table_type == "MANAGED" or table_type == "EXTERNAL":
-        return DatabricksRelationType.Table.value
+        return DatabricksRelationType.Table
     else:
         is_materialized = project.run_sql(
             "select is_materialized from `system`.`information_schema`.`views`"
@@ -25,9 +25,9 @@ def query_relation_type(project, relation: BaseRelation) -> Optional[str]:
             fetch="one",
         )[0]
         if is_materialized == "TRUE":
-            return DatabricksRelationType.MaterializedView.value
+            return DatabricksRelationType.MaterializedView
         else:
-            return DatabricksRelationType.View.value
+            return DatabricksRelationType.View
 
 
 streaming_table = """
diff --git a/tests/unit/api_client/api_test_base.py b/tests/unit/api_client/api_test_base.py
index f76b54a2..3d60dc04 100644
--- a/tests/unit/api_client/api_test_base.py
+++ b/tests/unit/api_client/api_test_base.py
@@ -1,8 +1,8 @@
 from typing import Any, Callable
+from unittest.mock import Mock
 
 import pytest
 from dbt_common.exceptions import DbtRuntimeError
-from mock import Mock
 
 
 class ApiTestBase:
diff --git a/tests/unit/api_client/test_cluster_api.py b/tests/unit/api_client/test_cluster_api.py
index 3a8efef8..fc47d221 100644
--- a/tests/unit/api_client/test_cluster_api.py
+++ b/tests/unit/api_client/test_cluster_api.py
@@ -1,7 +1,8 @@
+from unittest.mock import patch
+
 import freezegun
 import pytest
 from dbt_common.exceptions import DbtRuntimeError
-from mock import patch
 
 from dbt.adapters.databricks.api_client import ClusterApi
 from tests.unit.api_client.api_test_base import ApiTestBase
diff --git a/tests/unit/api_client/test_command_api.py b/tests/unit/api_client/test_command_api.py
index 880f054e..2021bd74 100644
--- a/tests/unit/api_client/test_command_api.py
+++ b/tests/unit/api_client/test_command_api.py
@@ -1,7 +1,8 @@
+from unittest.mock import Mock, patch
+
 import freezegun
 import pytest
 from dbt_common.exceptions import DbtRuntimeError
-from mock import Mock, patch
 
 from dbt.adapters.databricks.api_client import CommandApi, CommandExecution
 from tests.unit.api_client.api_test_base import ApiTestBase
diff --git a/tests/unit/api_client/test_command_context_api.py b/tests/unit/api_client/test_command_context_api.py
index a54e1c06..779e667b 100644
--- a/tests/unit/api_client/test_command_context_api.py
+++ b/tests/unit/api_client/test_command_context_api.py
@@ -1,5 +1,6 @@
+from unittest.mock import Mock
+
 import pytest
-from mock import Mock
 
 from dbt.adapters.databricks.api_client import CommandContextApi
 from tests.unit.api_client.api_test_base import ApiTestBase
diff --git a/tests/unit/api_client/test_job_runs_api.py b/tests/unit/api_client/test_job_runs_api.py
index adc280b5..9640136c 100644
--- a/tests/unit/api_client/test_job_runs_api.py
+++ b/tests/unit/api_client/test_job_runs_api.py
@@ -1,7 +1,8 @@
+from unittest.mock import patch
+
 import freezegun
 import pytest
 from dbt_common.exceptions import DbtRuntimeError
-from mock import patch
 
 from dbt.adapters.databricks.api_client import JobRunsApi
 from tests.unit.api_client.api_test_base import ApiTestBase
diff --git a/tests/unit/api_client/test_workspace_api.py b/tests/unit/api_client/test_workspace_api.py
index d981f6c2..322a9172 100644
--- a/tests/unit/api_client/test_workspace_api.py
+++ b/tests/unit/api_client/test_workspace_api.py
@@ -1,7 +1,7 @@
 import base64
+from unittest.mock import Mock
 
 import pytest
-from mock import Mock
 
 from dbt.adapters.databricks.api_client import WorkspaceApi
 from tests.unit.api_client.api_test_base import ApiTestBase
diff --git a/tests/unit/events/test_connection_events.py b/tests/unit/events/test_connection_events.py
index b0c8af10..81a30c1d 100644
--- a/tests/unit/events/test_connection_events.py
+++ b/tests/unit/events/test_connection_events.py
@@ -1,4 +1,4 @@
-from mock import Mock
+from unittest.mock import Mock
 
 from dbt.adapters.databricks.events.connection_events import (
     ConnectionAcquire,
diff --git a/tests/unit/events/test_cursor_events.py b/tests/unit/events/test_cursor_events.py
index 41248c56..e492415a 100644
--- a/tests/unit/events/test_cursor_events.py
+++ b/tests/unit/events/test_cursor_events.py
@@ -1,4 +1,4 @@
-from mock import Mock
+from unittest.mock import Mock
 
 from dbt.adapters.databricks.events.cursor_events import CursorCloseError, CursorEvent
 
diff --git a/tests/unit/macros/adapters/test_python_macros.py b/tests/unit/macros/adapters/test_python_macros.py
index 590dd1c5..bedc4812 100644
--- a/tests/unit/macros/adapters/test_python_macros.py
+++ b/tests/unit/macros/adapters/test_python_macros.py
@@ -1,5 +1,6 @@
+from unittest.mock import MagicMock
+
 import pytest
-from mock import MagicMock
 
 from tests.unit.macros.base import MacroTestBase
 
diff --git a/tests/unit/macros/base.py b/tests/unit/macros/base.py
index db5d6d5c..cb9c7f28 100644
--- a/tests/unit/macros/base.py
+++ b/tests/unit/macros/base.py
@@ -1,9 +1,9 @@
 import re
 from typing import Any
+from unittest.mock import Mock
 
 import pytest
 from jinja2 import Environment, FileSystemLoader, PackageLoader, Template
-from mock import Mock
 
 from dbt.adapters.databricks.relation import DatabricksRelation
 
diff --git a/tests/unit/macros/relations/test_view_macros.py b/tests/unit/macros/relations/test_view_macros.py
index 39f2912e..700e966a 100644
--- a/tests/unit/macros/relations/test_view_macros.py
+++ b/tests/unit/macros/relations/test_view_macros.py
@@ -1,5 +1,6 @@
+from unittest.mock import Mock
+
 import pytest
-from mock import Mock
 
 from tests.unit.macros.base import MacroTestBase
 
diff --git a/tests/unit/python/test_python_run_tracker.py b/tests/unit/python/test_python_run_tracker.py
index 8b022170..e841d463 100644
--- a/tests/unit/python/test_python_run_tracker.py
+++ b/tests/unit/python/test_python_run_tracker.py
@@ -1,4 +1,4 @@
-from mock import Mock
+from unittest.mock import Mock
 
 from dbt.adapters.databricks.python_models.run_tracking import PythonRunTracker
 
diff --git a/tests/unit/python/test_python_submitters.py b/tests/unit/python/test_python_submitters.py
index 33498308..cf707730 100644
--- a/tests/unit/python/test_python_submitters.py
+++ b/tests/unit/python/test_python_submitters.py
@@ -33,7 +33,9 @@ def compiled_code():
 @pytest.fixture
 def config_compiler():
     compiler = Mock()
-    compiler.compile.return_value = PythonJobDetails("name", {}, {})
+    compiler.compile.return_value = PythonJobDetails(
+        run_name="name", job_spec={}, additional_job_config={}
+    )
     return compiler
 
 
diff --git a/tests/unit/relation_configs/test_comment.py b/tests/unit/relation_configs/test_comment.py
index 1a43ba2f..71d80207 100644
--- a/tests/unit/relation_configs/test_comment.py
+++ b/tests/unit/relation_configs/test_comment.py
@@ -1,5 +1,6 @@
+from unittest.mock import Mock
+
 from agate import Table
-from mock import Mock
 
 from dbt.adapters.databricks.relation_configs.comment import CommentConfig, CommentProcessor
 
diff --git a/tests/unit/relation_configs/test_materialized_view_config.py b/tests/unit/relation_configs/test_materialized_view_config.py
index 21fa743d..09aa9571 100644
--- a/tests/unit/relation_configs/test_materialized_view_config.py
+++ b/tests/unit/relation_configs/test_materialized_view_config.py
@@ -1,5 +1,6 @@
+from unittest.mock import Mock
+
 from agate import Row, Table
-from mock import Mock
 
 from dbt.adapters.databricks.relation_configs.comment import CommentConfig
 from dbt.adapters.databricks.relation_configs.materialized_view import (
diff --git a/tests/unit/relation_configs/test_partitioning.py b/tests/unit/relation_configs/test_partitioning.py
index 8d6fcae4..fb345f06 100644
--- a/tests/unit/relation_configs/test_partitioning.py
+++ b/tests/unit/relation_configs/test_partitioning.py
@@ -1,4 +1,4 @@
-from mock import Mock
+from unittest.mock import Mock
 
 from dbt.adapters.databricks.relation_configs.partitioning import (
     PartitionedByConfig,
diff --git a/tests/unit/relation_configs/test_query.py b/tests/unit/relation_configs/test_query.py
index 87138c3c..3cd1f1b3 100644
--- a/tests/unit/relation_configs/test_query.py
+++ b/tests/unit/relation_configs/test_query.py
@@ -1,6 +1,7 @@
+from unittest.mock import Mock
+
 import pytest
 from agate import Row
-from mock import Mock
 
 from dbt.adapters.databricks.relation_configs.query import QueryConfig, QueryProcessor
 from dbt.exceptions import DbtRuntimeError
diff --git a/tests/unit/relation_configs/test_refresh.py b/tests/unit/relation_configs/test_refresh.py
index 17fdb2c8..9da56e4d 100644
--- a/tests/unit/relation_configs/test_refresh.py
+++ b/tests/unit/relation_configs/test_refresh.py
@@ -1,5 +1,6 @@
+from unittest.mock import Mock
+
 import pytest
-from mock import Mock
 
 from dbt.adapters.databricks.relation_configs.refresh import RefreshConfig, RefreshProcessor
 from dbt.exceptions import DbtRuntimeError
diff --git a/tests/unit/relation_configs/test_streaming_table_config.py b/tests/unit/relation_configs/test_streaming_table_config.py
index 7681fd68..2ad77763 100644
--- a/tests/unit/relation_configs/test_streaming_table_config.py
+++ b/tests/unit/relation_configs/test_streaming_table_config.py
@@ -1,4 +1,4 @@
-from mock import Mock
+from unittest.mock import Mock
 
 from dbt.adapters.databricks.relation_configs.comment import CommentConfig
 from dbt.adapters.databricks.relation_configs.partitioning import PartitionedByConfig
diff --git a/tests/unit/relation_configs/test_tags.py b/tests/unit/relation_configs/test_tags.py
index bc1ed229..e893d3f5 100644
--- a/tests/unit/relation_configs/test_tags.py
+++ b/tests/unit/relation_configs/test_tags.py
@@ -1,6 +1,7 @@
+from unittest.mock import Mock
+
 import pytest
 from agate import Table
-from mock import Mock
 
 from dbt.adapters.databricks.relation_configs.tags import TagsConfig, TagsProcessor
 from dbt.exceptions import DbtRuntimeError
diff --git a/tests/unit/relation_configs/test_tblproperties.py b/tests/unit/relation_configs/test_tblproperties.py
index 9fac55d0..dab860b7 100644
--- a/tests/unit/relation_configs/test_tblproperties.py
+++ b/tests/unit/relation_configs/test_tblproperties.py
@@ -1,5 +1,6 @@
+from unittest.mock import Mock
+
 import pytest
-from mock import Mock
 
 from dbt.adapters.databricks.relation_configs.tblproperties import (
     TblPropertiesConfig,
diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py
index 58bd396e..b26d5f80 100644
--- a/tests/unit/test_adapter.py
+++ b/tests/unit/test_adapter.py
@@ -1,11 +1,10 @@
 from multiprocessing import get_context
 from typing import Any, Optional
+from unittest.mock import Mock, patch
 
-import mock
 import pytest
 from agate import Row
 from dbt_common.exceptions import DbtConfigError, DbtValidationError
-from mock import Mock
 
 import dbt.flags as flags
 from dbt.adapters.databricks import DatabricksAdapter, __version__
@@ -115,7 +114,7 @@ def test_invalid_custom_user_agent(self):
         with pytest.raises(DbtValidationError) as excinfo:
             config = self._get_config()
             adapter = DatabricksAdapter(config, get_context("spawn"))
-            with mock.patch.dict("os.environ", **{DBT_DATABRICKS_INVOCATION_ENV: "(Some-thing)"}):
+            with patch.dict("os.environ", **{DBT_DATABRICKS_INVOCATION_ENV: "(Some-thing)"}):
                 connection = adapter.acquire_connection("dummy")
                 connection.handle  # trigger lazy-load
 
@@ -125,11 +124,11 @@ def test_custom_user_agent(self):
         config = self._get_config()
         adapter = DatabricksAdapter(config, get_context("spawn"))
 
-        with mock.patch(
+        with patch(
             "dbt.adapters.databricks.connections.dbsql.connect",
             new=self._connect_func(expected_invocation_env="databricks-workflows"),
         ):
-            with mock.patch.dict(
+            with patch.dict(
                 "os.environ", **{DBT_DATABRICKS_INVOCATION_ENV: "databricks-workflows"}
             ):
                 connection = adapter.acquire_connection("dummy")
@@ -187,11 +186,11 @@ def _test_environment_http_headers(
 
         adapter = DatabricksAdapter(config, get_context("spawn"))
 
-        with mock.patch(
+        with patch(
             "dbt.adapters.databricks.connections.dbsql.connect",
             new=self._connect_func(expected_http_headers=expected_http_headers),
         ):
-            with mock.patch.dict(
+            with patch.dict(
                 "os.environ",
                 **{DBT_DATABRICKS_HTTP_SESSION_HEADERS: http_headers_str},
             ):
@@ -204,7 +203,7 @@ def test_oauth_settings(self):
 
         adapter = DatabricksAdapter(config, get_context("spawn"))
 
-        with mock.patch(
+        with patch(
             "dbt.adapters.databricks.connections.dbsql.connect",
             new=self._connect_func(expected_no_token=True),
         ):
@@ -217,7 +216,7 @@ def test_client_creds_settings(self):
 
         adapter = DatabricksAdapter(config, get_context("spawn"))
 
-        with mock.patch(
+        with patch(
             "dbt.adapters.databricks.connections.dbsql.connect",
             new=self._connect_func(expected_client_creds=True),
         ):
@@ -277,7 +276,7 @@ def _test_databricks_sql_connector_connection(self, connect):
         config = self._get_config()
         adapter = DatabricksAdapter(config, get_context("spawn"))
 
-        with mock.patch("dbt.adapters.databricks.connections.dbsql.connect", new=connect):
+        with patch("dbt.adapters.databricks.connections.dbsql.connect", new=connect):
             connection = adapter.acquire_connection("dummy")
             connection.handle  # trigger lazy-load
 
@@ -301,7 +300,7 @@ def _test_databricks_sql_connector_catalog_connection(self, connect):
         config = self._get_config()
         adapter = DatabricksAdapter(config, get_context("spawn"))
 
-        with mock.patch("dbt.adapters.databricks.connections.dbsql.connect", new=connect):
+        with patch("dbt.adapters.databricks.connections.dbsql.connect", new=connect):
             connection = adapter.acquire_connection("dummy")
             connection.handle  # trigger lazy-load
 
@@ -328,7 +327,7 @@ def _test_databricks_sql_connector_http_header_connection(self, http_headers, co
         config = self._get_config(connection_parameters={"http_headers": http_headers})
         adapter = DatabricksAdapter(config, get_context("spawn"))
 
-        with mock.patch("dbt.adapters.databricks.connections.dbsql.connect", new=connect):
+        with patch("dbt.adapters.databricks.connections.dbsql.connect", new=connect):
             connection = adapter.acquire_connection("dummy")
             connection.handle  # trigger lazy-load
 
@@ -342,13 +341,13 @@ def _test_databricks_sql_connector_http_header_connection(self, http_headers, co
             assert connection.credentials.schema == "analytics"
 
     def test_list_relations_without_caching__no_relations(self):
-        with mock.patch.object(DatabricksAdapter, "get_relations_without_caching") as mocked:
+        with patch.object(DatabricksAdapter, "get_relations_without_caching") as mocked:
             mocked.return_value = []
             adapter = DatabricksAdapter(Mock(flags={}), get_context("spawn"))
             assert adapter.list_relations("database", "schema") == []
 
     def test_list_relations_without_caching__some_relations(self):
-        with mock.patch.object(DatabricksAdapter, "get_relations_without_caching") as mocked:
+        with patch.object(DatabricksAdapter, "get_relations_without_caching") as mocked:
             mocked.return_value = [("name", "table", "hudi", "owner")]
             adapter = DatabricksAdapter(Mock(flags={}), get_context("spawn"))
             relations = adapter.list_relations("database", "schema")
@@ -362,7 +361,7 @@ def test_list_relations_without_caching__some_relations(self):
             assert relation.is_hudi
 
     def test_list_relations_without_caching__hive_relation(self):
-        with mock.patch.object(DatabricksAdapter, "get_relations_without_caching") as mocked:
+        with patch.object(DatabricksAdapter, "get_relations_without_caching") as mocked:
             mocked.return_value = [("name", "table", None, None)]
             adapter = DatabricksAdapter(Mock(flags={}), get_context("spawn"))
             relations = adapter.list_relations("database", "schema")
@@ -375,18 +374,18 @@ def test_list_relations_without_caching__hive_relation(self):
             assert not relation.has_information()
 
     def test_get_schema_for_catalog__no_columns(self):
-        with mock.patch.object(DatabricksAdapter, "_list_relations_with_information") as list_info:
+        with patch.object(DatabricksAdapter, "_list_relations_with_information") as list_info:
             list_info.return_value = [(Mock(), "info")]
-            with mock.patch.object(DatabricksAdapter, "_get_columns_for_catalog") as get_columns:
+            with patch.object(DatabricksAdapter, "_get_columns_for_catalog") as get_columns:
                 get_columns.return_value = []
                 adapter = DatabricksAdapter(Mock(flags={}), get_context("spawn"))
                 table = adapter._get_schema_for_catalog("database", "schema", "name")
                 assert len(table.rows) == 0
 
     def test_get_schema_for_catalog__some_columns(self):
-        with mock.patch.object(DatabricksAdapter, "_list_relations_with_information") as list_info:
+        with patch.object(DatabricksAdapter, "_list_relations_with_information") as list_info:
             list_info.return_value = [(Mock(), "info")]
-            with mock.patch.object(DatabricksAdapter, "_get_columns_for_catalog") as get_columns:
+            with patch.object(DatabricksAdapter, "_get_columns_for_catalog") as get_columns:
                 get_columns.return_value = [
                     {"name": "col1", "type": "string", "comment": "comment"},
                     {"name": "col2", "type": "string", "comment": "comment"},
@@ -908,7 +907,7 @@ def test_describe_table_extended_2048_char_limit(self):
         assert get_identifier_list_string(table_names) == "|".join(table_names)
 
         # If environment variable is set, then limit the number of characters
-        with mock.patch.dict("os.environ", **{"DBT_DESCRIBE_TABLE_2048_CHAR_BYPASS": "true"}):
+        with patch.dict("os.environ", **{"DBT_DESCRIBE_TABLE_2048_CHAR_BYPASS": "true"}):
             # Long list of table names is capped
             assert get_identifier_list_string(table_names) == "*"
 
@@ -937,7 +936,7 @@ def test_describe_table_extended_should_limit(self):
         table_names = set([f"customers_{i}" for i in range(200)])
 
         # If environment variable is set, then limit the number of characters
-        with mock.patch.dict("os.environ", **{"DBT_DESCRIBE_TABLE_2048_CHAR_BYPASS": "true"}):
+        with patch.dict("os.environ", **{"DBT_DESCRIBE_TABLE_2048_CHAR_BYPASS": "true"}):
             # Long list of table names is capped
             assert get_identifier_list_string(table_names) == "*"
 
@@ -950,7 +949,7 @@ def test_describe_table_extended_may_limit(self):
         table_names = set([f"customers_{i}" for i in range(200)])
 
         # If environment variable is set, then we may limit the number of characters
-        with mock.patch.dict("os.environ", **{"DBT_DESCRIBE_TABLE_2048_CHAR_BYPASS": "true"}):
+        with patch.dict("os.environ", **{"DBT_DESCRIBE_TABLE_2048_CHAR_BYPASS": "true"}):
             # But a short list of table names is not capped
             assert get_identifier_list_string(list(table_names)[:5]) == "|".join(
                 list(table_names)[:5]
diff --git a/tests/unit/test_compute_config.py b/tests/unit/test_compute_config.py
index c6ad51bd..994d4ae9 100644
--- a/tests/unit/test_compute_config.py
+++ b/tests/unit/test_compute_config.py
@@ -1,6 +1,7 @@
+from unittest.mock import Mock
+
 import pytest
 from dbt_common.exceptions import DbtRuntimeError
-from mock import Mock
 
 from dbt.adapters.databricks import connections
 from dbt.adapters.databricks.credentials import DatabricksCredentials
diff --git a/tox.ini b/tox.ini
deleted file mode 100644
index e0ba45ea..00000000
--- a/tox.ini
+++ /dev/null
@@ -1,71 +0,0 @@
-[tox]
-skipsdist = True
-envlist = linter, unit
-
-[testenv:linter]
-basepython = python3
-commands =
-  {envpython} -m ruff format --check
-  {envpython} -m ruff check
-  {envpython} -m mypy --config-file mypy.ini --explicit-package-bases dbt tests
-passenv =
-  DBT_*
-  PYTEST_ADDOPTS
-deps =
-  -r{toxinidir}/dev-requirements.txt
-  -r{toxinidir}/requirements.txt
-
-[testenv:format]
-basepython = python3
-commands = {envpython} -m ruff format
-passenv =
-  DBT_*
-  PYTEST_ADDOPTS
-deps =
-  -r{toxinidir}/dev-requirements.txt
-
-[testenv:unit]
-basepython = python3
-commands = {envpython} -m pytest --color=yes -v {posargs} tests/unit
-passenv =
-  DBT_*
-  PYTEST_ADDOPTS
-deps =
-  -r{toxinidir}/dev-requirements.txt
-  -r{toxinidir}/requirements.txt
-
-[testenv:integration-databricks-cluster]
-basepython = python3
-commands =
-  /bin/bash -c '{envpython} -m pytest -v --profile databricks_cluster -n 10 tests/functional/adapter/* {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'
-passenv =
-  DBT_*
-  PYTEST_ADDOPTS
-deps =
-  -r{toxinidir}/dev-requirements.txt
-  -r{toxinidir}/requirements.txt
-allowlist_externals = /bin/bash
-
-[testenv:integration-databricks-uc-cluster]
-basepython = python3
-commands =
-  /bin/bash -c '{envpython} -m pytest -v --profile databricks_uc_cluster -n 10 tests/functional/adapter/* {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'
-passenv =
-  DBT_*
-  PYTEST_ADDOPTS
-deps =
-  -r{toxinidir}/dev-requirements.txt
-  -r{toxinidir}/requirements.txt
-allowlist_externals = /bin/bash
-
-[testenv:integration-databricks-uc-sql-endpoint]
-basepython = python3
-commands =
-  /bin/bash -c '{envpython} -m pytest -v --profile databricks_uc_sql_endpoint -n 10 --dist loadscope tests/functional/adapter/* {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'
-passenv =
-  DBT_*
-  PYTEST_ADDOPTS
-deps =
-  -r{toxinidir}/dev-requirements.txt
-  -r{toxinidir}/requirements.txt
-allowlist_externals = /bin/bash

From 63980336fb46423b9f7c0148451d3acfa324f96a Mon Sep 17 00:00:00 2001
From: Ben Cassell <98852248+benc-db@users.noreply.github.com>
Date: Tue, 26 Nov 2024 12:37:10 -0800
Subject: [PATCH 4/4] Fix Python regressions in 1.9.0beta (#857)

---
 dbt/adapters/databricks/api_client.py         | 16 ++++++++---
 dbt/adapters/databricks/behaviors/columns.py  |  2 --
 dbt/adapters/databricks/impl.py               |  2 +-
 .../python_models/python_submissions.py       |  7 +++++
 .../databricks/macros/adapters/columns.sql    | 27 +++++++++++++++++++
 .../macros/adapters/persist_docs.sql          | 27 -------------------
 .../adapter/python_model/fixtures.py          |  9 +++++++
 .../adapter/python_model/test_python_model.py | 19 +++++++++++++
 tests/unit/api_client/test_command_api.py     | 13 +++++++++
 9 files changed, 88 insertions(+), 34 deletions(-)
 create mode 100644 dbt/include/databricks/macros/adapters/columns.sql

diff --git a/dbt/adapters/databricks/api_client.py b/dbt/adapters/databricks/api_client.py
index c2dbefb1..c14af9f5 100644
--- a/dbt/adapters/databricks/api_client.py
+++ b/dbt/adapters/databricks/api_client.py
@@ -88,8 +88,10 @@ def start(self, cluster_id: str) -> None:
 
         response = self.session.post("/start", json={"cluster_id": cluster_id})
         if response.status_code != 200:
-            raise DbtRuntimeError(f"Error starting terminated cluster.\n {response.content!r}")
-        logger.debug(f"Cluster start response={response}")
+            if self.status(cluster_id) not in ["RUNNING", "PENDING"]:
+                raise DbtRuntimeError(f"Error starting terminated cluster.\n {response.content!r}")
+            else:
+                logger.debug("Presuming race condition, waiting for cluster to start")
 
         self.wait_for_cluster(cluster_id)
 
@@ -289,7 +291,7 @@ def cancel(self, command: CommandExecution) -> None:
             raise DbtRuntimeError(f"Cancel command {command} failed.\n {response.content!r}")
 
     def poll_for_completion(self, command: CommandExecution) -> None:
-        self._poll_api(
+        response = self._poll_api(
             url="/status",
             params={
                 "clusterId": command.cluster_id,
@@ -300,7 +302,13 @@ def poll_for_completion(self, command: CommandExecution) -> None:
             terminal_states={"Finished", "Error", "Cancelled"},
             expected_end_state="Finished",
             unexpected_end_state_func=self._get_exception,
-        )
+        ).json()
+
+        if response["results"]["resultType"] == "error":
+            raise DbtRuntimeError(
+                f"Python model failed with traceback as:\n"
+                f"{utils.remove_ansi(response['results']['cause'])}"
+            )
 
     def _get_exception(self, response: Response) -> None:
         response_json = response.json()
diff --git a/dbt/adapters/databricks/behaviors/columns.py b/dbt/adapters/databricks/behaviors/columns.py
index 63d80d2d..8f3d1eca 100644
--- a/dbt/adapters/databricks/behaviors/columns.py
+++ b/dbt/adapters/databricks/behaviors/columns.py
@@ -7,8 +7,6 @@
 from dbt.adapters.databricks.utils import handle_missing_objects
 from dbt.adapters.sql import SQLAdapter
 
-GET_COLUMNS_COMMENTS_MACRO_NAME = "get_columns_comments"
-
 
 class GetColumnsBehavior(ABC):
     @classmethod
diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py
index ad1b28da..3e0288b0 100644
--- a/dbt/adapters/databricks/impl.py
+++ b/dbt/adapters/databricks/impl.py
@@ -85,7 +85,7 @@
 SHOW_TABLE_EXTENDED_MACRO_NAME = "show_table_extended"
 SHOW_TABLES_MACRO_NAME = "show_tables"
 SHOW_VIEWS_MACRO_NAME = "show_views"
-GET_COLUMNS_COMMENTS_MACRO_NAME = "get_columns_comments"
+
 
 USE_INFO_SCHEMA_FOR_COLUMNS = BehaviorFlag(
     name="use_info_schema_for_columns",
diff --git a/dbt/adapters/databricks/python_models/python_submissions.py b/dbt/adapters/databricks/python_models/python_submissions.py
index cbb27b44..afcb383c 100644
--- a/dbt/adapters/databricks/python_models/python_submissions.py
+++ b/dbt/adapters/databricks/python_models/python_submissions.py
@@ -8,6 +8,7 @@
 from dbt.adapters.base import PythonJobHelper
 from dbt.adapters.databricks.api_client import CommandExecution, DatabricksApiClient, WorkflowJobApi
 from dbt.adapters.databricks.credentials import DatabricksCredentials
+from dbt.adapters.databricks.logging import logger
 from dbt.adapters.databricks.python_models.python_config import ParsedPythonModel
 from dbt.adapters.databricks.python_models.run_tracking import PythonRunTracker
 
@@ -70,6 +71,8 @@ def __init__(
 
     @override
     def submit(self, compiled_code: str) -> None:
+        logger.debug("Submitting Python model using the Command API.")
+
         context_id = self.api_client.command_contexts.create(self.cluster_id)
         command_exec: Optional[CommandExecution] = None
         try:
@@ -263,6 +266,8 @@ def create(
 
     @override
     def submit(self, compiled_code: str) -> None:
+        logger.debug("Submitting Python model using the Job Run API.")
+
         file_path = self.uploader.upload(compiled_code)
         job_config = self.config_compiler.compile(file_path)
 
@@ -494,6 +499,8 @@ def create(
 
     @override
     def submit(self, compiled_code: str) -> None:
+        logger.debug("Submitting Python model using the Workflow API.")
+
         file_path = self.uploader.upload(compiled_code)
 
         workflow_config, existing_job_id = self.config_compiler.compile(file_path)
diff --git a/dbt/include/databricks/macros/adapters/columns.sql b/dbt/include/databricks/macros/adapters/columns.sql
new file mode 100644
index 00000000..e1fc1d11
--- /dev/null
+++ b/dbt/include/databricks/macros/adapters/columns.sql
@@ -0,0 +1,27 @@
+
+{% macro get_columns_comments(relation) -%}
+  {% call statement('get_columns_comments', fetch_result=True) -%}
+    describe table {{ relation|lower }}
+  {% endcall %}
+
+  {% do return(load_result('get_columns_comments').table) %}
+{% endmacro %}
+
+{% macro get_columns_comments_via_information_schema(relation) -%}
+  {% call statement('repair_table', fetch_result=False) -%}
+    REPAIR TABLE {{ relation|lower }} SYNC METADATA
+  {% endcall %}
+  {% call statement('get_columns_comments_via_information_schema', fetch_result=True) -%}
+    select
+      column_name,
+      full_data_type,
+      comment
+    from `system`.`information_schema`.`columns`
+    where
+      table_catalog = '{{ relation.database|lower }}' and
+      table_schema = '{{ relation.schema|lower }}' and 
+      table_name = '{{ relation.identifier|lower }}'
+  {% endcall %}
+
+  {% do return(load_result('get_columns_comments_via_information_schema').table) %}
+{% endmacro %}
diff --git a/dbt/include/databricks/macros/adapters/persist_docs.sql b/dbt/include/databricks/macros/adapters/persist_docs.sql
index f623a8a6..8e959a9f 100644
--- a/dbt/include/databricks/macros/adapters/persist_docs.sql
+++ b/dbt/include/databricks/macros/adapters/persist_docs.sql
@@ -20,33 +20,6 @@
   {% do run_query(comment_query) %}
 {% endmacro %}
 
-{% macro get_columns_comments(relation) -%}
-  {% call statement('get_columns_comments', fetch_result=True) -%}
-    describe table {{ relation|lower }}
-  {% endcall %}
-
-  {% do return(load_result('get_columns_comments').table) %}
-{% endmacro %}
-
-{% macro get_columns_comments_via_information_schema(relation) -%}
-  {% call statement('repair_table', fetch_result=False) -%}
-    REPAIR TABLE {{ relation|lower }} SYNC METADATA
-  {% endcall %}
-  {% call statement('get_columns_comments_via_information_schema', fetch_result=True) -%}
-    select
-      column_name,
-      full_data_type,
-      comment
-    from `system`.`information_schema`.`columns`
-    where
-      table_catalog = '{{ relation.database|lower }}' and
-      table_schema = '{{ relation.schema|lower }}' and 
-      table_name = '{{ relation.identifier|lower }}'
-  {% endcall %}
-
-  {% do return(load_result('get_columns_comments_via_information_schema').table) %}
-{% endmacro %}
-
 {% macro databricks__persist_docs(relation, model, for_relation, for_columns) -%}
   {%- if for_relation and config.persist_relation_docs() and model.description %}
     {% do alter_table_comment(relation, model) %}
diff --git a/tests/functional/adapter/python_model/fixtures.py b/tests/functional/adapter/python_model/fixtures.py
index 5ce51702..127bcf74 100644
--- a/tests/functional/adapter/python_model/fixtures.py
+++ b/tests/functional/adapter/python_model/fixtures.py
@@ -9,6 +9,15 @@ def model(dbt, spark):
     return spark.createDataFrame(data, schema=['test', 'test2'])
 """
 
+python_error_model = """
+import pandas as pd
+
+def model(dbt, spark):
+    raise Exception("This is an error")
+
+    return pd.DataFrame()
+"""
+
 serverless_schema = """version: 2
 
 models:
diff --git a/tests/functional/adapter/python_model/test_python_model.py b/tests/functional/adapter/python_model/test_python_model.py
index 482674fe..858214b7 100644
--- a/tests/functional/adapter/python_model/test_python_model.py
+++ b/tests/functional/adapter/python_model/test_python_model.py
@@ -17,6 +17,25 @@ class TestPythonModel(BasePythonModelTests):
     pass
 
 
+@pytest.mark.python
+@pytest.mark.skip_profile("databricks_uc_sql_endpoint")
+class TestPythonFailureModel:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {"my_failure_model.py": override_fixtures.python_error_model}
+
+    def test_failure_model(self, project):
+        util.run_dbt(["run"], expect_pass=False)
+
+
+@pytest.mark.python
+@pytest.mark.skip_profile("databricks_uc_sql_endpoint")
+class TestPythonFailureModelNotebook(TestPythonFailureModel):
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {"models": {"+create_notebook": "true"}}
+
+
 @pytest.mark.python
 @pytest.mark.skip_profile("databricks_uc_sql_endpoint")
 class TestPythonIncrementalModel(BasePythonIncrementalTests):
diff --git a/tests/unit/api_client/test_command_api.py b/tests/unit/api_client/test_command_api.py
index 2021bd74..90efaf47 100644
--- a/tests/unit/api_client/test_command_api.py
+++ b/tests/unit/api_client/test_command_api.py
@@ -86,6 +86,7 @@ def test_poll_for_completion__200(self, _, api, session, host, execution):
         session.get.return_value.status_code = 200
         session.get.return_value.json.return_value = {
             "status": "Finished",
+            "results": {"resultType": "finished"},
         }
 
         api.poll_for_completion(execution)
@@ -99,3 +100,15 @@ def test_poll_for_completion__200(self, _, api, session, host, execution):
             },
             json=None,
         )
+
+    @freezegun.freeze_time("2020-01-01")
+    @patch("dbt.adapters.databricks.api_client.time.sleep")
+    def test_poll_for_completion__200_with_error(self, _, api, session, host, execution):
+        session.get.return_value.status_code = 200
+        session.get.return_value.json.return_value = {
+            "status": "Finished",
+            "results": {"resultType": "error", "cause": "race condition"},
+        }
+
+        with pytest.raises(DbtRuntimeError, match="Python model failed"):
+            api.poll_for_completion(execution)