From 6be80ad3804ea06677279b64f4c94686c5a78d98 Mon Sep 17 00:00:00 2001 From: Leah Antkiewicz Date: Sun, 1 May 2022 11:35:36 -0400 Subject: [PATCH 1/6] Updating to test python 3.10 --- .github/scripts/integration-test-matrix.js | 2 +- .github/workflows/main.yml | 4 ++-- dev-requirements.txt | 2 +- setup.py | 1 + tests/integration/base.py | 8 ++++++++ tox.ini | 6 +++--- 6 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 6a33653a2..1e3bb0f0d 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -1,6 +1,6 @@ module.exports = ({ context }) => { const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.7", "3.8", "3.9"]; + const supportedPythonVersions = ["3.7", "3.8", "3.9", "3.10"]; const supportedAdapters = ["bigquery"]; // if PR, generate matrix based on files changed and PR labels diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d2ed07654..2e961dfa7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,7 +69,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8] # TODO: support unit testing for python 3.9 (https://github.com/dbt-labs/dbt/issues/3689) + python-version: ['3.7', '3.8', '3.9', '3.10'] env: TOXENV: "unit" @@ -173,7 +173,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.7, 3.8, 3.9] + python-version: ['3.7', '3.8', '3.9', '3.10'] steps: - name: Set up Python ${{ matrix.python-version }} diff --git a/dev-requirements.txt b/dev-requirements.txt index ff7b6522b..fd49e3110 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -9,7 +9,7 @@ flake8 flaky freezegun==1.1.0 ipdb -mypy==0.782 +mypy==0.942 pip-tools pre-commit pytest diff --git a/setup.py b/setup.py index 461e9746b..0adf95f3b 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ def _get_dbt_core_version(): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], python_requires=">=3.7", ) diff --git a/tests/integration/base.py b/tests/integration/base.py index 243ac92d8..ec6769e8e 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -7,6 +7,7 @@ import tempfile import traceback import unittest +import warnings from contextlib import contextmanager from datetime import datetime from functools import wraps @@ -230,6 +231,13 @@ def _generate_test_root_dir(self): return normalize(tempfile.mkdtemp(prefix='dbt-int-test-')) def setUp(self): + # Logbook warnings are ignored so we don't have to fork logbook to support python 3.10. + # This _only_ works for tests in `test/integration`. + warnings.filterwarnings( + "ignore", + category=DeprecationWarning, + module="logbook" + ) self.dbt_core_install_root = os.path.dirname(dbt.__file__) log_manager.reset_handlers() self.initial_dir = INITIAL_ROOT diff --git a/tox.ini b/tox.ini index a5da0d0a6..348be15af 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] skipsdist = True -envlist = py37,py38,py39 +envlist = py37,py38,py39,py310 -[testenv:{unit,py37,py38,py39,py}] +[testenv:{unit,py37,py38,py39,py310,py}] description = unit testing skip_install = true passenv = DBT_* PYTEST_ADDOPTS @@ -11,7 +11,7 @@ deps = -rdev-requirements.txt -e. -[testenv:{integration,py37,py38,py39,py}-{bigquery}] +[testenv:{integration,py37,py38,py39,py310,py}-{bigquery}] description = adapter plugin integration testing skip_install = true passenv = DBT_* BIGQUERY_TEST_* PYTEST_ADDOPTS From a6d52d37d83bdcda4fedccced8572f2faecc40a7 Mon Sep 17 00:00:00 2001 From: Leah Antkiewicz Date: Wed, 4 May 2022 22:28:53 -0400 Subject: [PATCH 2/6] Fixing mypy issues from upgrade --- .pre-commit-config.yaml | 2 +- dbt/__init__.py | 0 dbt/adapters/bigquery/impl.py | 2 +- mypy.ini | 3 +- third-party-stubs/agate/__init__.pyi | 89 +++++++++++++++++++++++++ third-party-stubs/agate/data_types.pyi | 71 ++++++++++++++++++++ third-party-stubs/requests/__init__.pyi | 1 + 7 files changed, 165 insertions(+), 3 deletions(-) create mode 100644 dbt/__init__.py create mode 100644 third-party-stubs/agate/__init__.pyi create mode 100644 third-party-stubs/agate/data_types.pyi create mode 100644 third-party-stubs/requests/__init__.pyi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d247581b..cf4ddf68a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: alias: flake8-check stages: [manual] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.782 + rev: v0.942 hooks: - id: mypy args: [--show-error-codes, --ignore-missing-imports] diff --git a/dbt/__init__.py b/dbt/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 50ca3c6e1..344ffb22e 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -296,7 +296,7 @@ def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str: @classmethod def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: - decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) + decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) # type: ignore[attr-defined] return "float64" if decimals else "int64" @classmethod diff --git a/mypy.ini b/mypy.ini index 51fada1b1..b111482fc 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,3 +1,4 @@ [mypy] -mypy_path = ./third-party-stubs +mypy_path = third-party-stubs/ namespace_packages = True +exclude = third-party-stubs/* diff --git a/third-party-stubs/agate/__init__.pyi b/third-party-stubs/agate/__init__.pyi new file mode 100644 index 000000000..c773cc7d7 --- /dev/null +++ b/third-party-stubs/agate/__init__.pyi @@ -0,0 +1,89 @@ +from collections.abc import Sequence + +from typing import Any, Optional, Callable, Iterable, Dict, Union + +from . import data_types as data_types +from .data_types import ( + Text as Text, + Number as Number, + Boolean as Boolean, + DateTime as DateTime, + Date as Date, + TimeDelta as TimeDelta, +) + +class MappedSequence(Sequence): + def __init__(self, values: Any, keys: Optional[Any] = ...) -> None: ... + def __unicode__(self): ... + def __getitem__(self, key: Any): ... + def __setitem__(self, key: Any, value: Any) -> None: ... + def __iter__(self): ... + def __len__(self): ... + def __eq__(self, other: Any): ... + def __ne__(self, other: Any): ... + def __contains__(self, value: Any): ... + def keys(self): ... + def values(self): ... + def items(self): ... + def get(self, key: Any, default: Optional[Any] = ...): ... + def dict(self): ... + +class Row(MappedSequence): ... + +class Table: + def __init__( + self, + rows: Any, + column_names: Optional[Any] = ..., + column_types: Optional[Any] = ..., + row_names: Optional[Any] = ..., + _is_fork: bool = ..., + ) -> None: ... + def __len__(self): ... + def __iter__(self): ... + def __getitem__(self, key: Any): ... + @property + def column_types(self): ... + @property + def column_names(self): ... + @property + def row_names(self): ... + @property + def columns(self): ... + @property + def rows(self): ... + def print_csv(self, **kwargs: Any) -> None: ... + def print_json(self, **kwargs: Any) -> None: ... + def where(self, test: Callable[[Row], bool]) -> "Table": ... + def select(self, key: Union[Iterable[str], str]) -> "Table": ... + # these definitions are much narrower than what's actually accepted + @classmethod + def from_object( + cls, obj: Iterable[Dict[str, Any]], *, column_types: Optional["TypeTester"] = None + ) -> "Table": ... + @classmethod + def from_csv( + cls, path: Iterable[str], *, column_types: Optional["TypeTester"] = None + ) -> "Table": ... + @classmethod + def merge(cls, tables: Iterable["Table"]) -> "Table": ... + def rename( + self, + column_names: Optional[Iterable[str]] = None, + row_names: Optional[Any] = None, + slug_columns: bool = False, + slug_rows: bool = False, + **kwargs: Any, + ) -> "Table": ... + +class TypeTester: + def __init__( + self, force: Any = ..., limit: Optional[Any] = ..., types: Optional[Any] = ... + ) -> None: ... + def run(self, rows: Any, column_names: Any): ... + +class MaxPrecision: + def __init__(self, column_name: Any) -> None: ... + +# this is not strictly true, but it's all we care about. +def aggregate(self, aggregations: MaxPrecision) -> int: ... diff --git a/third-party-stubs/agate/data_types.pyi b/third-party-stubs/agate/data_types.pyi new file mode 100644 index 000000000..8114f7b55 --- /dev/null +++ b/third-party-stubs/agate/data_types.pyi @@ -0,0 +1,71 @@ +from typing import Any, Optional + +DEFAULT_NULL_VALUES: Any + +class DataType: + null_values: Any = ... + def __init__(self, null_values: Any = ...) -> None: ... + def test(self, d: Any): ... + def cast(self, d: Any) -> None: ... + def csvify(self, d: Any): ... + def jsonify(self, d: Any): ... + +DEFAULT_TRUE_VALUES: Any +DEFAULT_FALSE_VALUES: Any + +class Boolean(DataType): + true_values: Any = ... + false_values: Any = ... + def __init__( + self, true_values: Any = ..., false_values: Any = ..., null_values: Any = ... + ) -> None: ... + def cast(self, d: Any): ... + def jsonify(self, d: Any): ... + +ZERO_DT: Any + +class Date(DataType): + date_format: Any = ... + parser: Any = ... + def __init__(self, date_format: Optional[Any] = ..., **kwargs: Any) -> None: ... + def cast(self, d: Any): ... + def csvify(self, d: Any): ... + def jsonify(self, d: Any): ... + +class DateTime(DataType): + datetime_format: Any = ... + timezone: Any = ... + def __init__( + self, datetime_format: Optional[Any] = ..., timezone: Optional[Any] = ..., **kwargs: Any + ) -> None: ... + def cast(self, d: Any): ... + def csvify(self, d: Any): ... + def jsonify(self, d: Any): ... + +DEFAULT_CURRENCY_SYMBOLS: Any +POSITIVE: Any +NEGATIVE: Any + +class Number(DataType): + locale: Any = ... + currency_symbols: Any = ... + group_symbol: Any = ... + decimal_symbol: Any = ... + def __init__( + self, + locale: str = ..., + group_symbol: Optional[Any] = ..., + decimal_symbol: Optional[Any] = ..., + currency_symbols: Any = ..., + **kwargs: Any, + ) -> None: ... + def cast(self, d: Any): ... + def jsonify(self, d: Any): ... + +class TimeDelta(DataType): + def cast(self, d: Any): ... + +class Text(DataType): + cast_nulls: Any = ... + def __init__(self, cast_nulls: bool = ..., **kwargs: Any) -> None: ... + def cast(self, d: Any): ... diff --git a/third-party-stubs/requests/__init__.pyi b/third-party-stubs/requests/__init__.pyi new file mode 100644 index 000000000..3bde133be --- /dev/null +++ b/third-party-stubs/requests/__init__.pyi @@ -0,0 +1 @@ +from requests.exceptions import ConnectionError as ConnectionError From a4396b45cdbc8d13e18a0841b31eaa1a84539343 Mon Sep 17 00:00:00 2001 From: Leah Antkiewicz Date: Thu, 5 May 2022 15:22:25 -0400 Subject: [PATCH 3/6] Extending path in init --- dbt/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbt/__init__.py b/dbt/__init__.py index e69de29bb..9088ea6ae 100644 --- a/dbt/__init__.py +++ b/dbt/__init__.py @@ -0,0 +1,3 @@ +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) \ No newline at end of file From bd57ad876c8bb5624b74ff686a527168cf0bb64e Mon Sep 17 00:00:00 2001 From: Leah Antkiewicz Date: Thu, 5 May 2022 17:46:32 -0400 Subject: [PATCH 4/6] Adding newline --- dbt/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/__init__.py b/dbt/__init__.py index 9088ea6ae..b36383a61 100644 --- a/dbt/__init__.py +++ b/dbt/__init__.py @@ -1,3 +1,3 @@ from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) \ No newline at end of file +__path__ = extend_path(__path__, __name__) From 10b1c57c5336bdc9a6574a1519ee07d7e2e45037 Mon Sep 17 00:00:00 2001 From: Leah Antkiewicz Date: Thu, 5 May 2022 17:53:25 -0400 Subject: [PATCH 5/6] Updating changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 528cd08b1..be153ce60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ -## dbt-bigquery 1.1.0 (Release TBD) +## dbt-bigquery 1.2.0 (Release TBD) +- Adding Python 3.10 testing and enabling mypy ([#177](https://github.com/dbt-labs/dbt-bigquery/pull/177)) + +## dbt-bigquery 1.1.0 (April 28, 2022) + +## dbt-bigquery 1.1.0rc2 (April 20, 2022) ### Fixes - Restore default behavior for query timeout. Set default `job_execution_timeout` to `None` by default. Keep 300 seconds as query timeout where previously used. From 7ae1e66486bcc9edfef4ee339330adb98e28c9db Mon Sep 17 00:00:00 2001 From: Leah Antkiewicz Date: Fri, 6 May 2022 19:21:51 -0400 Subject: [PATCH 6/6] Adding response stubs --- dev-requirements.txt | 1 + third-party-stubs/requests/__init__.pyi | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 third-party-stubs/requests/__init__.pyi diff --git a/dev-requirements.txt b/dev-requirements.txt index fd49e3110..09cf7f7c2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -19,5 +19,6 @@ pytest-csv pytest-xdist pytz tox>=3.13 +types-requests twine wheel diff --git a/third-party-stubs/requests/__init__.pyi b/third-party-stubs/requests/__init__.pyi deleted file mode 100644 index 3bde133be..000000000 --- a/third-party-stubs/requests/__init__.pyi +++ /dev/null @@ -1 +0,0 @@ -from requests.exceptions import ConnectionError as ConnectionError