diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 6a33653a2..1e3bb0f0d 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -1,6 +1,6 @@ module.exports = ({ context }) => { const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.7", "3.8", "3.9"]; + const supportedPythonVersions = ["3.7", "3.8", "3.9", "3.10"]; const supportedAdapters = ["bigquery"]; // if PR, generate matrix based on files changed and PR labels diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d2ed07654..2e961dfa7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,7 +69,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8] # TODO: support unit testing for python 3.9 (https://github.com/dbt-labs/dbt/issues/3689) + python-version: ['3.7', '3.8', '3.9', '3.10'] env: TOXENV: "unit" @@ -173,7 +173,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.7, 3.8, 3.9] + python-version: ['3.7', '3.8', '3.9', '3.10'] steps: - name: Set up Python ${{ matrix.python-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d247581b..cf4ddf68a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: alias: flake8-check stages: [manual] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.782 + rev: v0.942 hooks: - id: mypy args: [--show-error-codes, --ignore-missing-imports] diff --git a/CHANGELOG.md b/CHANGELOG.md index 528cd08b1..be153ce60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ -## dbt-bigquery 1.1.0 (Release TBD) +## dbt-bigquery 1.2.0 (Release TBD) +- Adding Python 3.10 testing and enabling mypy ([#177](https://github.com/dbt-labs/dbt-bigquery/pull/177)) + +## dbt-bigquery 1.1.0 (April 28, 2022) + +## dbt-bigquery 1.1.0rc2 (April 20, 2022) ### Fixes - Restore default behavior for query timeout. Set default `job_execution_timeout` to `None` by default. Keep 300 seconds as query timeout where previously used. diff --git a/dbt/__init__.py b/dbt/__init__.py new file mode 100644 index 000000000..b36383a61 --- /dev/null +++ b/dbt/__init__.py @@ -0,0 +1,3 @@ +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 50ca3c6e1..344ffb22e 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -296,7 +296,7 @@ def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str: @classmethod def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: - decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) + decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) # type: ignore[attr-defined] return "float64" if decimals else "int64" @classmethod diff --git a/dev-requirements.txt b/dev-requirements.txt index ff7b6522b..09cf7f7c2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -9,7 +9,7 @@ flake8 flaky freezegun==1.1.0 ipdb -mypy==0.782 +mypy==0.942 pip-tools pre-commit pytest @@ -19,5 +19,6 @@ pytest-csv pytest-xdist pytz tox>=3.13 +types-requests twine wheel diff --git a/mypy.ini b/mypy.ini index 51fada1b1..b111482fc 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,3 +1,4 @@ [mypy] -mypy_path = ./third-party-stubs +mypy_path = third-party-stubs/ namespace_packages = True +exclude = third-party-stubs/* diff --git a/setup.py b/setup.py index 461e9746b..0adf95f3b 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ def _get_dbt_core_version(): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], python_requires=">=3.7", ) diff --git a/tests/integration/base.py b/tests/integration/base.py index 243ac92d8..ec6769e8e 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -7,6 +7,7 @@ import tempfile import traceback import unittest +import warnings from contextlib import contextmanager from datetime import datetime from functools import wraps @@ -230,6 +231,13 @@ def _generate_test_root_dir(self): return normalize(tempfile.mkdtemp(prefix='dbt-int-test-')) def setUp(self): + # Logbook warnings are ignored so we don't have to fork logbook to support python 3.10. + # This _only_ works for tests in `test/integration`. + warnings.filterwarnings( + "ignore", + category=DeprecationWarning, + module="logbook" + ) self.dbt_core_install_root = os.path.dirname(dbt.__file__) log_manager.reset_handlers() self.initial_dir = INITIAL_ROOT diff --git a/third-party-stubs/agate/__init__.pyi b/third-party-stubs/agate/__init__.pyi new file mode 100644 index 000000000..c773cc7d7 --- /dev/null +++ b/third-party-stubs/agate/__init__.pyi @@ -0,0 +1,89 @@ +from collections.abc import Sequence + +from typing import Any, Optional, Callable, Iterable, Dict, Union + +from . import data_types as data_types +from .data_types import ( + Text as Text, + Number as Number, + Boolean as Boolean, + DateTime as DateTime, + Date as Date, + TimeDelta as TimeDelta, +) + +class MappedSequence(Sequence): + def __init__(self, values: Any, keys: Optional[Any] = ...) -> None: ... + def __unicode__(self): ... + def __getitem__(self, key: Any): ... + def __setitem__(self, key: Any, value: Any) -> None: ... + def __iter__(self): ... + def __len__(self): ... + def __eq__(self, other: Any): ... + def __ne__(self, other: Any): ... + def __contains__(self, value: Any): ... + def keys(self): ... + def values(self): ... + def items(self): ... + def get(self, key: Any, default: Optional[Any] = ...): ... + def dict(self): ... + +class Row(MappedSequence): ... + +class Table: + def __init__( + self, + rows: Any, + column_names: Optional[Any] = ..., + column_types: Optional[Any] = ..., + row_names: Optional[Any] = ..., + _is_fork: bool = ..., + ) -> None: ... + def __len__(self): ... + def __iter__(self): ... + def __getitem__(self, key: Any): ... + @property + def column_types(self): ... + @property + def column_names(self): ... + @property + def row_names(self): ... + @property + def columns(self): ... + @property + def rows(self): ... + def print_csv(self, **kwargs: Any) -> None: ... + def print_json(self, **kwargs: Any) -> None: ... + def where(self, test: Callable[[Row], bool]) -> "Table": ... + def select(self, key: Union[Iterable[str], str]) -> "Table": ... + # these definitions are much narrower than what's actually accepted + @classmethod + def from_object( + cls, obj: Iterable[Dict[str, Any]], *, column_types: Optional["TypeTester"] = None + ) -> "Table": ... + @classmethod + def from_csv( + cls, path: Iterable[str], *, column_types: Optional["TypeTester"] = None + ) -> "Table": ... + @classmethod + def merge(cls, tables: Iterable["Table"]) -> "Table": ... + def rename( + self, + column_names: Optional[Iterable[str]] = None, + row_names: Optional[Any] = None, + slug_columns: bool = False, + slug_rows: bool = False, + **kwargs: Any, + ) -> "Table": ... + +class TypeTester: + def __init__( + self, force: Any = ..., limit: Optional[Any] = ..., types: Optional[Any] = ... + ) -> None: ... + def run(self, rows: Any, column_names: Any): ... + +class MaxPrecision: + def __init__(self, column_name: Any) -> None: ... + +# this is not strictly true, but it's all we care about. +def aggregate(self, aggregations: MaxPrecision) -> int: ... diff --git a/third-party-stubs/agate/data_types.pyi b/third-party-stubs/agate/data_types.pyi new file mode 100644 index 000000000..8114f7b55 --- /dev/null +++ b/third-party-stubs/agate/data_types.pyi @@ -0,0 +1,71 @@ +from typing import Any, Optional + +DEFAULT_NULL_VALUES: Any + +class DataType: + null_values: Any = ... + def __init__(self, null_values: Any = ...) -> None: ... + def test(self, d: Any): ... + def cast(self, d: Any) -> None: ... + def csvify(self, d: Any): ... + def jsonify(self, d: Any): ... + +DEFAULT_TRUE_VALUES: Any +DEFAULT_FALSE_VALUES: Any + +class Boolean(DataType): + true_values: Any = ... + false_values: Any = ... + def __init__( + self, true_values: Any = ..., false_values: Any = ..., null_values: Any = ... + ) -> None: ... + def cast(self, d: Any): ... + def jsonify(self, d: Any): ... + +ZERO_DT: Any + +class Date(DataType): + date_format: Any = ... + parser: Any = ... + def __init__(self, date_format: Optional[Any] = ..., **kwargs: Any) -> None: ... + def cast(self, d: Any): ... + def csvify(self, d: Any): ... + def jsonify(self, d: Any): ... + +class DateTime(DataType): + datetime_format: Any = ... + timezone: Any = ... + def __init__( + self, datetime_format: Optional[Any] = ..., timezone: Optional[Any] = ..., **kwargs: Any + ) -> None: ... + def cast(self, d: Any): ... + def csvify(self, d: Any): ... + def jsonify(self, d: Any): ... + +DEFAULT_CURRENCY_SYMBOLS: Any +POSITIVE: Any +NEGATIVE: Any + +class Number(DataType): + locale: Any = ... + currency_symbols: Any = ... + group_symbol: Any = ... + decimal_symbol: Any = ... + def __init__( + self, + locale: str = ..., + group_symbol: Optional[Any] = ..., + decimal_symbol: Optional[Any] = ..., + currency_symbols: Any = ..., + **kwargs: Any, + ) -> None: ... + def cast(self, d: Any): ... + def jsonify(self, d: Any): ... + +class TimeDelta(DataType): + def cast(self, d: Any): ... + +class Text(DataType): + cast_nulls: Any = ... + def __init__(self, cast_nulls: bool = ..., **kwargs: Any) -> None: ... + def cast(self, d: Any): ... diff --git a/tox.ini b/tox.ini index a5da0d0a6..348be15af 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] skipsdist = True -envlist = py37,py38,py39 +envlist = py37,py38,py39,py310 -[testenv:{unit,py37,py38,py39,py}] +[testenv:{unit,py37,py38,py39,py310,py}] description = unit testing skip_install = true passenv = DBT_* PYTEST_ADDOPTS @@ -11,7 +11,7 @@ deps = -rdev-requirements.txt -e. -[testenv:{integration,py37,py38,py39,py}-{bigquery}] +[testenv:{integration,py37,py38,py39,py310,py}-{bigquery}] description = adapter plugin integration testing skip_install = true passenv = DBT_* BIGQUERY_TEST_* PYTEST_ADDOPTS