From 448d8d7509dac8f6c1971fc128af4da2e95dffa4 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 8 Nov 2021 20:29:31 +0100 Subject: [PATCH 1/3] Try branch, see if this fixes --- dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index a44d72a4e..26c5fd1bf 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,6 +1,6 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@only_warn_for_missing_patch_node#egg=dbt-core&subdirectory=core freezegun==0.3.9 pytest==6.0.2 From af5f1b4fbc728bc369dd145d915caac0eeabbadc Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 8 Nov 2021 20:37:39 +0100 Subject: [PATCH 2/3] Update tests/unit/utils.py per dbt-core#4212 --- tests/unit/utils.py | 224 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 212 insertions(+), 12 deletions(-) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 89124d3be..68d0fc33f 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -1,18 +1,20 @@ """Unit test utility functions. - Note that all imports should be inside the functions to avoid import/mocking issues. """ +import string import os from unittest import mock from unittest import TestCase -from hologram import ValidationError +import agate +import pytest +from dbt.dataclass_schema import ValidationError +from dbt.config.project import PartialProject def normalize(path): """On windows, neither is enough on its own: - >>> normcase('C:\\documents/ALL CAPS/subdir\\..') 'c:\\documents\\all caps\\subdir\\..' >>> normpath('C:\\documents/ALL CAPS/subdir\\..') @@ -28,9 +30,10 @@ class Obj: single_threaded = False -def mock_connection(name): +def mock_connection(name, state='open'): conn = mock.MagicMock() conn.name = name + conn.state = state return conn @@ -42,7 +45,7 @@ def profile_from_dict(profile, profile_name, cli_vars='{}'): if not isinstance(cli_vars, dict): cli_vars = parse_cli_vars(cli_vars) - renderer = ProfileRenderer(generate_base_context(cli_vars)) + renderer = ProfileRenderer(cli_vars) return Profile.from_raw_profile_info( profile, profile_name, @@ -58,13 +61,18 @@ def project_from_dict(project, profile, packages=None, selectors=None, cli_vars= if not isinstance(cli_vars, dict): cli_vars = parse_cli_vars(cli_vars) - renderer = DbtProjectYamlRenderer(generate_target_context(profile, cli_vars)) + renderer = DbtProjectYamlRenderer(profile, cli_vars) project_root = project.pop('project-root', os.getcwd()) - return Project.render_from_dict( - project_root, project, packages, selectors, renderer - ) + partial = PartialProject.from_dicts( + project_root=project_root, + project_dict=project, + packages_dict=packages, + selectors_dict=selectors, + ) + return partial.render(renderer) + def config_from_parts_or_dicts(project, profile, packages=None, selectors=None, cli_vars='{}'): @@ -108,6 +116,14 @@ def inject_plugin(plugin): FACTORY.plugins[key] = plugin +def inject_plugin_for(config): + # from dbt.adapters.postgres import Plugin, PostgresAdapter + from dbt.adapters.factory import FACTORY + FACTORY.load_plugin(config.credentials.type) + adapter = FACTORY.get_adapter(config) + return adapter + + def inject_adapter(value, plugin): """Inject the given adapter into the adapter factory, so your hand-crafted artisanal adapter will be available from get_adapter() as if dbt loaded it. @@ -118,6 +134,13 @@ def inject_adapter(value, plugin): FACTORY.adapters[key] = value +def clear_plugin(plugin): + from dbt.adapters.factory import FACTORY + key = plugin.adapter.type() + FACTORY.plugins.pop(key, None) + FACTORY.adapters.pop(key, None) + + class ContractTestCase(TestCase): ContractType = None @@ -126,11 +149,12 @@ def setUp(self): super().setUp() def assert_to_dict(self, obj, dct): - self.assertEqual(obj.to_dict(), dct) + self.assertEqual(obj.to_dict(omit_none=True), dct) def assert_from_dict(self, obj, dct, cls=None): if cls is None: cls = self.ContractType + cls.validate(dct) self.assertEqual(cls.from_dict(dct), obj) def assert_symmetric(self, obj, dct, cls=None): @@ -142,9 +166,59 @@ def assert_fails_validation(self, dct, cls=None): cls = self.ContractType with self.assertRaises(ValidationError): + cls.validate(dct) cls.from_dict(dct) +def compare_dicts(dict1, dict2): + first_set = set(dict1.keys()) + second_set = set(dict2.keys()) + print(f"--- Difference between first and second keys: {first_set.difference(second_set)}") + print(f"--- Difference between second and first keys: {second_set.difference(first_set)}") + common_keys = set(first_set).intersection(set(second_set)) + found_differences = False + for key in common_keys: + if dict1[key] != dict2[key] : + print(f"--- --- first dict: {key}: {str(dict1[key])}") + print(f"--- --- second dict: {key}: {str(dict2[key])}") + found_differences = True + if found_differences: + print("--- Found differences in dictionaries") + else: + print("--- Found no differences in dictionaries") + + +def assert_from_dict(obj, dct, cls=None): + if cls is None: + cls = obj.__class__ + cls.validate(dct) + obj_from_dict = cls.from_dict(dct) + if hasattr(obj, 'created_at'): + obj_from_dict.created_at = 1 + obj.created_at = 1 + assert obj_from_dict == obj + + +def assert_to_dict(obj, dct): + obj_to_dict = obj.to_dict(omit_none=True) + if 'created_at' in obj_to_dict: + obj_to_dict['created_at'] = 1 + if 'created_at' in dct: + dct['created_at'] = 1 + assert obj_to_dict == dct + + +def assert_symmetric(obj, dct, cls=None): + assert_to_dict(obj, dct) + assert_from_dict(obj, dct, cls) + + +def assert_fails_validation(dct, cls): + with pytest.raises(ValidationError): + cls.validate(dct) + cls.from_dict(dct) + + def generate_name_macros(package): from dbt.contracts.graph.parsed import ParsedMacro from dbt.node_types import NodeType @@ -158,7 +232,6 @@ def generate_name_macros(package): sql = f'{{% macro {name}(value, node) %}} {{% if value %}} {{{{ value }}}} {{% else %}} {{{{ {source} }}}} {{% endif %}} {{% endmacro %}}' name_sql[name] = sql - all_sql = '\n'.join(name_sql.values()) for name, sql in name_sql.items(): pm = ParsedMacro( name=name, @@ -168,7 +241,134 @@ def generate_name_macros(package): original_file_path=normalize('macros/macro.sql'), root_path='./dbt_packages/root', path=normalize('macros/macro.sql'), - raw_sql=all_sql, macro_sql=sql, ) yield pm + + +class TestAdapterConversions(TestCase): + def _get_tester_for(self, column_type): + from dbt.clients import agate_helper + if column_type is agate.TimeDelta: # dbt never makes this! + return agate.TimeDelta() + + for instance in agate_helper.DEFAULT_TYPE_TESTER._possible_types: + if type(instance) is column_type: + return instance + + raise ValueError(f'no tester for {column_type}') + + def _make_table_of(self, rows, column_types): + column_names = list(string.ascii_letters[:len(rows[0])]) + if isinstance(column_types, type): + column_types = [self._get_tester_for(column_types) for _ in column_names] + else: + column_types = [self._get_tester_for(typ) for typ in column_types] + table = agate.Table(rows, column_names=column_names, column_types=column_types) + return table + + +def MockMacro(package, name='my_macro', **kwargs): + from dbt.contracts.graph.parsed import ParsedMacro + from dbt.node_types import NodeType + + mock_kwargs = dict( + resource_type=NodeType.Macro, + package_name=package, + unique_id=f'macro.{package}.{name}', + original_file_path='/dev/null', + ) + + mock_kwargs.update(kwargs) + + macro = mock.MagicMock( + spec=ParsedMacro, + **mock_kwargs + ) + macro.name = name + return macro + + +def MockMaterialization(package, name='my_materialization', adapter_type=None, **kwargs): + if adapter_type is None: + adapter_type = 'default' + kwargs['adapter_type'] = adapter_type + return MockMacro(package, f'materialization_{name}_{adapter_type}', **kwargs) + + +def MockGenerateMacro(package, component='some_component', **kwargs): + name = f'generate_{component}_name' + return MockMacro(package, name=name, **kwargs) + + +def MockSource(package, source_name, name, **kwargs): + from dbt.node_types import NodeType + from dbt.contracts.graph.parsed import ParsedSourceDefinition + src = mock.MagicMock( + __class__=ParsedSourceDefinition, + resource_type=NodeType.Source, + source_name=source_name, + package_name=package, + unique_id=f'source.{package}.{source_name}.{name}', + search_name=f'{source_name}.{name}', + **kwargs + ) + src.name = name + return src + + +def MockNode(package, name, resource_type=None, **kwargs): + from dbt.node_types import NodeType + from dbt.contracts.graph.parsed import ParsedModelNode, ParsedSeedNode + if resource_type is None: + resource_type = NodeType.Model + if resource_type == NodeType.Model: + cls = ParsedModelNode + elif resource_type == NodeType.Seed: + cls = ParsedSeedNode + else: + raise ValueError(f'I do not know how to handle {resource_type}') + node = mock.MagicMock( + __class__=cls, + resource_type=resource_type, + package_name=package, + unique_id=f'{str(resource_type)}.{package}.{name}', + search_name=name, + **kwargs + ) + node.name = name + return node + + +def MockDocumentation(package, name, **kwargs): + from dbt.node_types import NodeType + from dbt.contracts.graph.parsed import ParsedDocumentation + doc = mock.MagicMock( + __class__=ParsedDocumentation, + resource_type=NodeType.Documentation, + package_name=package, + search_name=name, + unique_id=f'{package}.{name}', + **kwargs + ) + doc.name = name + return doc + + +def load_internal_manifest_macros(config, macro_hook = lambda m: None): + from dbt.parser.manifest import ManifestLoader + return ManifestLoader.load_macros(config, macro_hook) + + + +def dict_replace(dct, **kwargs): + dct = dct.copy() + dct.update(kwargs) + return dct + + +def replace_config(n, **kwargs): + return n.replace( + config=n.config.replace(**kwargs), + unrendered_config=dict_replace(n.unrendered_config, **kwargs), + ) From 90951888887f66c5dd01373a0584ae7554f025db Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 8 Nov 2021 20:41:45 +0100 Subject: [PATCH 3/3] Circle: run integration only, and altogether --- .circleci/config.yml | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 99154fb64..5646bcea5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,14 +1,6 @@ version: 2.1 jobs: - unit: - environment: - DBT_INVOCATION_ENV: circle - docker: - - image: fishtownanalytics/test-container:10 - steps: - - checkout - - run: tox -e flake8,unit integration-spark-thrift: environment: @@ -94,16 +86,7 @@ workflows: version: 2 test-everything: jobs: - - unit - - integration-spark-thrift: - requires: - - unit - - integration-spark-databricks-http: - requires: - - integration-spark-thrift - - integration-spark-databricks-odbc-cluster: - requires: - - integration-spark-thrift - - integration-spark-databricks-odbc-endpoint: - requires: - - integration-spark-thrift + - integration-spark-thrift + - integration-spark-databricks-http + - integration-spark-databricks-odbc-cluster + - integration-spark-databricks-odbc-endpoint