Add support for Python 3.11 (#6326)

* Get running with Python 3.11

* More tests passing, mypy still unhappy

* Upgrade to 3.11, and bump mashumaro

* patch importlib.import_module last

* lambda: Policy() default_factory on include and quote policy

* Add changelog entry

* Put a lambda on it

* Fix text formatting for log file

* Handle variant type return from e.log_level()

Co-authored-by: Jeremy Cohen <jeremy@dbtlabs.com>
Co-authored-by: Josh Taylor <joshuataylorx@gmail.com>
Co-authored-by: Michelle Ark <michelle.ark@dbtlabs.com>

.changes/unreleased/Features-20221206-150704.yaml

@@ -0,0 +1,7 @@
+kind: Features
+body: Add support for Python 3.11
+time: 2022-12-06T15:07:04.753127+01:00
+custom:
+  Author: joshuataylor MichelleArk jtcohen6
+  Issue: "6147"
+  PR: "6326"

.github/workflows/main.yml

@@ -73,7 +73,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     env:
       TOXENV: "unit"
@@ -118,7 +118,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
         os: [ubuntu-20.04]
         include:
           - python-version: 3.8

CONTRIBUTING.md

@@ -56,7 +56,7 @@ There are some tools that will be helpful to you in developing locally. While th
 
 These are the tools used in `dbt-core` development and testing:
 
-- [`tox`](https://tox.readthedocs.io/en/latest/) to manage virtualenvs across python versions. We currently target the latest patch releases for Python 3.7, 3.8, 3.9, and 3.10
+- [`tox`](https://tox.readthedocs.io/en/latest/) to manage virtualenvs across python versions. We currently target the latest patch releases for Python 3.7, 3.8, 3.9, 3.10 and 3.11
 - [`pytest`](https://docs.pytest.org/en/latest/) to define, discover, and run tests
 - [`flake8`](https://flake8.pycqa.org/en/latest/) for code linting
 - [`black`](https://github.com/psf/black) for code formatting
@@ -160,7 +160,7 @@ suites.
 
 #### `tox`
 
-[`tox`](https://tox.readthedocs.io/en/latest/) takes care of managing virtualenvs and install dependencies in order to run tests. You can also run tests in parallel, for example, you can run unit tests for Python 3.7, Python 3.8, Python 3.9, and Python 3.10 checks in parallel with `tox -p`. Also, you can run unit tests for specific python versions with `tox -e py37`. The configuration for these tests in located in `tox.ini`.
+[`tox`](https://tox.readthedocs.io/en/latest/) takes care of managing virtualenvs and install dependencies in order to run tests. You can also run tests in parallel, for example, you can run unit tests for Python 3.7, Python 3.8, Python 3.9, Python 3.10 and Python 3.11 checks in parallel with `tox -p`. Also, you can run unit tests for specific python versions with `tox -e py37`. The configuration for these tests in located in `tox.ini`.
 
 #### `pytest`
 

Dockerfile.test

@@ -49,6 +49,9 @@ RUN apt-get update \
     python3.10 \
     python3.10-dev \
     python3.10-venv \
+    python3.11 \
+    python3.11-dev \
+    python3.11-venv \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 

core/dbt/adapters/base/relation.py

@@ -1,5 +1,5 @@
 from collections.abc import Hashable
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Optional, TypeVar, Any, Type, Dict, Union, Iterator, Tuple, Set
 
 from dbt.contracts.graph.nodes import SourceDefinition, ParsedNode
@@ -26,8 +26,10 @@ class BaseRelation(FakeAPIObject, Hashable):
     path: Path
     type: Optional[RelationType] = None
     quote_character: str = '"'
-    include_policy: Policy = Policy()
-    quote_policy: Policy = Policy()
+    # Python 3.11 requires that these use default_factory instead of simple default
+    # ValueError: mutable default <class 'dbt.contracts.relation.Policy'> for field include_policy is not allowed: use default_factory
+    include_policy: Policy = field(default_factory=lambda: Policy())
+    quote_policy: Policy = field(default_factory=lambda: Policy())
     dbt_created: bool = False
 
     def _is_exactish_match(self, field: ComponentName, value: str) -> bool:
@@ -38,9 +40,9 @@ def _is_exactish_match(self, field: ComponentName, value: str) -> bool:
 
     @classmethod
     def _get_field_named(cls, field_name):
-        for field, _ in cls._get_fields():
-            if field.name == field_name:
-                return field
+        for f, _ in cls._get_fields():
+            if f.name == field_name:
+                return f
         # this should be unreachable
         raise ValueError(f"BaseRelation has no {field_name} field!")
 
@@ -51,11 +53,11 @@ def __eq__(self, other):
 
     @classmethod
     def get_default_quote_policy(cls) -> Policy:
-        return cls._get_field_named("quote_policy").default
+        return cls._get_field_named("quote_policy").default_factory()
 
     @classmethod
     def get_default_include_policy(cls) -> Policy:
-        return cls._get_field_named("include_policy").default
+        return cls._get_field_named("include_policy").default_factory()
 
     def get(self, key, default=None):
         """Override `.get` to return a metadata object so we don't break

core/dbt/contracts/project.py

@@ -222,7 +222,7 @@ class Project(HyphenatedDbtClassMixin, Replaceable):
         ),
     )
     packages: List[PackageSpec] = field(default_factory=list)
-    query_comment: Optional[Union[QueryComment, NoValue, str]] = NoValue()
+    query_comment: Optional[Union[QueryComment, NoValue, str]] = field(default_factory=NoValue)
 
     @classmethod
     def validate(cls, data):

core/dbt/events/eventmgr.py

@@ -128,7 +128,11 @@ def create_debug_line(self, e: BaseEvent) -> str:
             log_line = f"\n\n{separator} {datetime.utcnow()} | {self.event_manager.invocation_id} {separator}\n"
         ts: str = datetime.utcnow().strftime("%H:%M:%S.%f")
         scrubbed_msg: str = self.scrubber(e.message())  # type: ignore
-        log_line += f"{self._get_color_tag()}{ts} [{e.log_level():<5}]{self._get_thread_name()} {scrubbed_msg}"
+        # log_level() for DynamicLevel events returns str instead of EventLevel
+        level = e.log_level().value if isinstance(e.log_level(), EventLevel) else e.log_level()
+        log_line += (
+            f"{self._get_color_tag()}{ts} [{level:<5}]{self._get_thread_name()} {scrubbed_msg}"
+        )
         return log_line
 
     def _get_color_tag(self) -> str:

core/dbt/events/types.py

@@ -1914,6 +1914,7 @@ def message(self) -> str:
     @classmethod
     def status_to_level(cls, status):
         # The statuses come from TestStatus
+        # TODO should this return EventLevel enum instead?
         level_lookup = {
             "fail": "error",
             "pass": "info",
@@ -2043,6 +2044,7 @@ def message(self) -> str:
     @classmethod
     def status_to_level(cls, status):
         # The statuses come from FreshnessStatus
+        # TODO should this return EventLevel enum instead?
         level_lookup = {
             "runtime error": "error",
             "pass": "info",

core/dbt/helper_types.py

@@ -3,7 +3,7 @@
 # necessary for annotating constructors
 from __future__ import annotations
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import timedelta
 from pathlib import Path
 from typing import Tuple, AbstractSet, Union
@@ -85,7 +85,7 @@ def __eq__(self, other):
 class NoValue(dbtClassMixin):
     """Sometimes, you want a way to say none that isn't None"""
 
-    novalue: NVEnum = NVEnum.novalue
+    novalue: NVEnum = field(default_factory=lambda: NVEnum.novalue)
 
 
 dbtClassMixin.register_field_encoders(

core/setup.py

@@ -81,6 +81,7 @@
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
     python_requires=">=3.7.2",
 )

plugins/postgres/setup.py

@@ -83,6 +83,7 @@ def _dbt_psycopg2_name():
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
     python_requires=">=3.7",
 )

tests/unit/test_version.py

@@ -673,10 +673,16 @@ def mock_import(*args, **kwargs):
 
 def mock_versions(mocker, installed="1.0.0", latest=None, plugins={}):
     mocker.patch("dbt.version.__version__", installed)
-    mock_plugins(mocker, plugins)
     mock_latest_versions(mocker, latest, plugins)
+    # mock_plugins must be called last to avoid erronously raising an ImportError.
+    mock_plugins(mocker, plugins)
 
 
+# NOTE: mock_plugins patches importlib.import_module, and should always be the last
+# patch to be mocked in order to avoid erronously raising an ImportError.
+# Explanation: As of Python 3.11, mock.patch indirectly uses importlib.import_module
+# and thus uses the mocked object (in this case, mock_import) instead of the real
+# implementation in subsequent mock.patch calls. Issue: https://github.com/python/cpython/issues/98771
 def mock_plugins(mocker, plugins):
     mock_find_spec = mocker.patch("importlib.util.find_spec")
     path = "/tmp/dbt/adapters"

tox.ini

@@ -2,7 +2,7 @@
 skipsdist = True
 envlist = unit,integration
 
-[testenv:{unit,py37,py38,py39,py310,py}]
+[testenv:{unit,py37,py38,py39,py310,py311,py}]
 description = unit testing
 download = true
 skip_install = true
@@ -16,7 +16,7 @@ deps =
   -rdev-requirements.txt
   -reditable-requirements.txt
 
-[testenv:{integration,py37-integration,py38-integration,py39-integration,py310-integration,py-integration}]
+[testenv:{integration,py37-integration,py38-integration,py39-integration,py310-integration,py311-integration,py-integration}]
 description = functional testing
 download = true
 skip_install = true