diff --git a/.changes/unreleased/Features-20231106-194752.yaml b/.changes/unreleased/Features-20231106-194752.yaml new file mode 100644 index 00000000000..2ea6553d339 --- /dev/null +++ b/.changes/unreleased/Features-20231106-194752.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add support of csv file fixtures to unit testing +time: 2023-11-06T19:47:52.501495-06:00 +custom: + Author: emmyoop + Issue: "8290" diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 71b71f720ff..e2b7312bc7c 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -652,6 +652,13 @@ def generic_test_paths(self): generic_test_paths.append(os.path.join(test_path, "generic")) return generic_test_paths + @property + def fixture_paths(self): + fixture_paths = [] + for test_path in self.test_paths: + fixture_paths.append(os.path.join(test_path, "fixtures")) + return fixture_paths + def __str__(self): cfg = self.to_project_config(with_packages=True) return str(cfg) diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index da64296e487..9baa175e2a6 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -4,6 +4,7 @@ from io import StringIO from dbt import deprecations +from dbt.clients.system import find_matching from dbt.node_types import NodeType from dbt.contracts.graph.semantic_models import ( Defaults, @@ -784,42 +785,73 @@ def format(self) -> UnitTestFormat: return UnitTestFormat.Dict @property - def rows(self) -> Union[str, List[Dict[str, Any]]]: - return [] + def rows(self) -> Optional[Union[str, List[Dict[str, Any]]]]: + return None + + @property + def fixture(self) -> Optional[str]: + return None - def get_rows(self) -> List[Dict[str, Any]]: + def get_rows(self, project_root: str, paths: List[str]) -> List[Dict[str, Any]]: if self.format == UnitTestFormat.Dict: assert isinstance(self.rows, List) return self.rows elif self.format == UnitTestFormat.CSV: - assert isinstance(self.rows, str) - dummy_file = StringIO(self.rows) - reader = csv.DictReader(dummy_file) rows = [] - for row in reader: - rows.append(row) + if self.fixture is not None: + assert isinstance(self.fixture, str) + file_path = self.get_fixture_path(self.fixture, project_root, paths) + with open(file_path, newline="") as csvfile: + reader = csv.DictReader(csvfile) + for row in reader: + rows.append(row) + else: # using inline csv + assert isinstance(self.rows, str) + dummy_file = StringIO(self.rows) + reader = csv.DictReader(dummy_file) + rows = [] + for row in reader: + rows.append(row) return rows + def get_fixture_path(self, fixture: str, project_root: str, paths: List[str]) -> str: + fixture_path = f"{fixture}.csv" + matches = find_matching(project_root, paths, fixture_path) + if len(matches) == 0: + raise ParsingError(f"Could not find fixture file {fixture} for unit test") + elif len(matches) > 1: + raise ParsingError( + f"Found multiple fixture files named {fixture} at {[d['relative_path'] for d in matches]}. Please use a unique name for each fixture file." + ) + + return matches[0]["absolute_path"] + def validate_fixture(self, fixture_type, test_name) -> None: - if (self.format == UnitTestFormat.Dict and not isinstance(self.rows, list)) or ( - self.format == UnitTestFormat.CSV and not isinstance(self.rows, str) - ): + if self.format == UnitTestFormat.Dict and not isinstance(self.rows, list): raise ParsingError( f"Unit test {test_name} has {fixture_type} rows which do not match format {self.format}" ) + if self.format == UnitTestFormat.CSV and not ( + isinstance(self.rows, str) or isinstance(self.fixture, str) + ): + raise ParsingError( + f"Unit test {test_name} has {fixture_type} rows or fixtures which do not match format {self.format}. Expected string." + ) @dataclass class UnitTestInputFixture(dbtClassMixin, UnitTestFixture): input: str - rows: Union[str, List[Dict[str, Any]]] = "" + rows: Optional[Union[str, List[Dict[str, Any]]]] = None format: UnitTestFormat = UnitTestFormat.Dict + fixture: Optional[str] = None @dataclass class UnitTestOutputFixture(dbtClassMixin, UnitTestFixture): - rows: Union[str, List[Dict[str, Any]]] = "" + rows: Optional[Union[str, List[Dict[str, Any]]]] = None format: UnitTestFormat = UnitTestFormat.Dict + fixture: Optional[str] = None @dataclass diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index a2bdb8852d8..c93f70b2997 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -67,7 +67,10 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): original_file_path=test_case.original_file_path, unique_id=test_case.unique_id, config=UnitTestNodeConfig( - materialized="unit", expected_rows=test_case.expect.get_rows() + materialized="unit", + expected_rows=test_case.expect.get_rows( + self.root_project.project_root, self.root_project.fixture_paths + ), ), raw_code=tested_node.raw_code, database=tested_node.database, @@ -122,7 +125,10 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): input_unique_id = f"model.{package_name}.{input_name}" input_node = ModelNode( raw_code=self._build_fixture_raw_code( - given.get_rows(), original_input_node_columns + given.get_rows( + self.root_project.project_root, self.root_project.fixture_paths + ), + original_input_node_columns, ), resource_type=NodeType.Model, package_name=package_name, diff --git a/tests/functional/unit_testing/fixtures.py b/tests/functional/unit_testing/fixtures.py new file mode 100644 index 00000000000..9018407baa9 --- /dev/null +++ b/tests/functional/unit_testing/fixtures.py @@ -0,0 +1,538 @@ +my_model_vars_sql = """ +SELECT +a+b as c, +concat(string_a, string_b) as string_c, +not_testing, date_a, +{{ dbt.string_literal(type_numeric()) }} as macro_call, +{{ dbt.string_literal(var('my_test')) }} as var_call, +{{ dbt.string_literal(env_var('MY_TEST', 'default')) }} as env_var_call, +{{ dbt.string_literal(invocation_id) }} as invocation_id +FROM {{ ref('my_model_a')}} my_model_a +JOIN {{ ref('my_model_b' )}} my_model_b +ON my_model_a.id = my_model_b.id +""" + +my_model_sql = """ +SELECT +a+b as c, +concat(string_a, string_b) as string_c, +not_testing, date_a +FROM {{ ref('my_model_a')}} my_model_a +JOIN {{ ref('my_model_b' )}} my_model_b +ON my_model_a.id = my_model_b.id +""" + +my_model_a_sql = """ +SELECT +1 as a, +1 as id, +2 as not_testing, +'a' as string_a, +DATE '2020-01-02' as date_a +""" + +my_model_b_sql = """ +SELECT +2 as b, +1 as id, +2 as c, +'b' as string_b +""" + +test_my_model_yml = """ +unit_tests: + - name: test_my_model + model: my_model + given: + - input: ref('my_model_a') + rows: + - {id: 1, a: 1} + - input: ref('my_model_b') + rows: + - {id: 1, b: 2} + - {id: 2, b: 2} + expect: + rows: + - {c: 2} + + - name: test_my_model_empty + model: my_model + given: + - input: ref('my_model_a') + rows: [] + - input: ref('my_model_b') + rows: + - {id: 1, b: 2} + - {id: 2, b: 2} + expect: + rows: [] + + - name: test_my_model_overrides + model: my_model + given: + - input: ref('my_model_a') + rows: + - {id: 1, a: 1} + - input: ref('my_model_b') + rows: + - {id: 1, b: 2} + - {id: 2, b: 2} + overrides: + macros: + type_numeric: override + invocation_id: 123 + vars: + my_test: var_override + env_vars: + MY_TEST: env_var_override + expect: + rows: + - {macro_call: override, var_call: var_override, env_var_call: env_var_override, invocation_id: 123} + + - name: test_my_model_string_concat + model: my_model + given: + - input: ref('my_model_a') + rows: + - {id: 1, string_a: a} + - input: ref('my_model_b') + rows: + - {id: 1, string_b: b} + expect: + rows: + - {string_c: ab} + config: + tags: test_this +""" + +datetime_test = """ + - name: test_my_model_datetime + model: my_model + given: + - input: ref('my_model_a') + rows: + - {id: 1, date_a: "2020-01-01"} + - input: ref('my_model_b') + rows: + - {id: 1} + expect: + rows: + - {date_a: "2020-01-01"} +""" + +event_sql = """ +select DATE '2020-01-01' as event_time, 1 as event +union all +select DATE '2020-01-02' as event_time, 2 as event +union all +select DATE '2020-01-03' as event_time, 3 as event +""" + +datetime_test_invalid_format_key = """ + - name: test_my_model_datetime + model: my_model + given: + - input: ref('my_model_a') + format: xxxx + rows: + - {id: 1, date_a: "2020-01-01"} + - input: ref('my_model_b') + rows: + - {id: 1} + expect: + rows: + - {date_a: "2020-01-01"} +""" + +datetime_test_invalid_csv_values = """ + - name: test_my_model_datetime + model: my_model + given: + - input: ref('my_model_a') + format: csv + rows: + - {id: 1, date_a: "2020-01-01"} + - input: ref('my_model_b') + rows: + - {id: 1} + expect: + rows: + - {date_a: "2020-01-01"} +""" + +datetime_test_invalid_csv_file_values = """ + - name: test_my_model_datetime + model: my_model + given: + - input: ref('my_model_a') + format: csv + rows: + - {id: 1, date_a: "2020-01-01"} + - input: ref('my_model_b') + rows: + - {id: 1} + expect: + rows: + - {date_a: "2020-01-01"} +""" + +event_sql = """ +select DATE '2020-01-01' as event_time, 1 as event +union all +select DATE '2020-01-02' as event_time, 2 as event +union all +select DATE '2020-01-03' as event_time, 3 as event +""" + +my_incremental_model_sql = """ +{{ + config( + materialized='incremental' + ) +}} + +select * from {{ ref('events') }} +{% if is_incremental() %} +where event_time > (select max(event_time) from {{ this }}) +{% endif %} +""" + +test_my_model_incremental_yml = """ +unit_tests: + - name: incremental_false + model: my_incremental_model + overrides: + macros: + is_incremental: false + given: + - input: ref('events') + rows: + - {event_time: "2020-01-01", event: 1} + expect: + rows: + - {event_time: "2020-01-01", event: 1} + - name: incremental_true + model: my_incremental_model + overrides: + macros: + is_incremental: true + given: + - input: ref('events') + rows: + - {event_time: "2020-01-01", event: 1} + - {event_time: "2020-01-02", event: 2} + - {event_time: "2020-01-03", event: 3} + - input: this + rows: + - {event_time: "2020-01-01", event: 1} + expect: + rows: + - {event_time: "2020-01-02", event: 2} + - {event_time: "2020-01-03", event: 3} +""" + +# -- inline csv tests + +test_my_model_csv_yml = """ +unit_tests: + - name: test_my_model + model: my_model + given: + - input: ref('my_model_a') + format: csv + rows: | + id,a + 1,1 + - input: ref('my_model_b') + format: csv + rows: | + id,b + 1,2 + 2,2 + expect: + format: csv + rows: | + c + 2 + + - name: test_my_model_empty + model: my_model + given: + - input: ref('my_model_a') + rows: [] + - input: ref('my_model_b') + format: csv + rows: | + id,b + 1,2 + 2,2 + expect: + rows: [] + - name: test_my_model_overrides + model: my_model + given: + - input: ref('my_model_a') + format: csv + rows: | + id,a + 1,1 + - input: ref('my_model_b') + format: csv + rows: | + id,b + 1,2 + 2,2 + overrides: + macros: + type_numeric: override + invocation_id: 123 + vars: + my_test: var_override + env_vars: + MY_TEST: env_var_override + expect: + rows: + - {macro_call: override, var_call: var_override, env_var_call: env_var_override, invocation_id: 123} + - name: test_my_model_string_concat + model: my_model + given: + - input: ref('my_model_a') + format: csv + rows: | + id,string_a + 1,a + - input: ref('my_model_b') + format: csv + rows: | + id,string_b + 1,b + expect: + format: csv + rows: | + string_c + ab + config: + tags: test_this +""" + +# -- csv file tests +test_my_model_file_csv_yml = """ +unit_tests: + - name: test_my_model + model: my_model + given: + - input: ref('my_model_a') + format: csv + fixture: test_my_model_a_numeric_fixture + - input: ref('my_model_b') + format: csv + fixture: test_my_model_fixture + expect: + format: csv + fixture: test_my_model_basic_fixture + + - name: test_my_model_empty + model: my_model + given: + - input: ref('my_model_a') + format: csv + fixture: test_my_model_a_empty_fixture + - input: ref('my_model_b') + format: csv + fixture: test_my_model_fixture + expect: + format: csv + fixture: test_my_model_a_empty_fixture + + - name: test_my_model_overrides + model: my_model + given: + - input: ref('my_model_a') + format: csv + fixture: test_my_model_a_numeric_fixture + - input: ref('my_model_b') + format: csv + fixture: test_my_model_fixture + overrides: + macros: + type_numeric: override + invocation_id: 123 + vars: + my_test: var_override + env_vars: + MY_TEST: env_var_override + expect: + rows: + - {macro_call: override, var_call: var_override, env_var_call: env_var_override, invocation_id: 123} + + - name: test_my_model_string_concat + model: my_model + given: + - input: ref('my_model_a') + format: csv + fixture: test_my_model_a_fixture + - input: ref('my_model_b') + format: csv + fixture: test_my_model_b_fixture + expect: + format: csv + fixture: test_my_model_concat_fixture + config: + tags: test_this +""" + +test_my_model_fixture_csv = """ +id,b +1,2 +2,2 +""" + +test_my_model_a_fixture_csv = """ +id,string_a +1,a +""" + +test_my_model_a_empty_fixture_csv = """ +""" + +test_my_model_a_numeric_fixture_csv = """ +id,a +1,1 +""" + +test_my_model_b_fixture_csv = """ +id,string_b +1,b +""" +test_my_model_basic_fixture_csv = """ +c +2 +""" + +test_my_model_concat_fixture_csv = """ +string_c +ab +""" + +# -- mixed inline and file csv +test_my_model_mixed_csv_yml = """ +unit_tests: + - name: test_my_model + model: my_model + given: + - input: ref('my_model_a') + format: csv + rows: | + id,a + 1,1 + - input: ref('my_model_b') + format: csv + rows: | + id,b + 1,2 + 2,2 + expect: + format: csv + fixture: test_my_model_basic_fixture + + - name: test_my_model_empty + model: my_model + given: + - input: ref('my_model_a') + format: csv + fixture: test_my_model_a_empty_fixture + - input: ref('my_model_b') + format: csv + rows: | + id,b + 1,2 + 2,2 + expect: + format: csv + fixture: test_my_model_a_empty_fixture + + - name: test_my_model_overrides + model: my_model + given: + - input: ref('my_model_a') + format: csv + rows: | + id,a + 1,1 + - input: ref('my_model_b') + format: csv + fixture: test_my_model_fixture + overrides: + macros: + type_numeric: override + invocation_id: 123 + vars: + my_test: var_override + env_vars: + MY_TEST: env_var_override + expect: + rows: + - {macro_call: override, var_call: var_override, env_var_call: env_var_override, invocation_id: 123} + + - name: test_my_model_string_concat + model: my_model + given: + - input: ref('my_model_a') + format: csv + fixture: test_my_model_a_fixture + - input: ref('my_model_b') + format: csv + fixture: test_my_model_b_fixture + expect: + format: csv + rows: | + string_c + ab + config: + tags: test_this +""" + +# unit tests with errors + +# -- fixture file doesn't exist +test_my_model_missing_csv_yml = """ +unit_tests: + - name: test_missing_csv_file + model: my_model + given: + - input: ref('my_model_a') + format: csv + rows: | + id,a + 1,1 + - input: ref('my_model_b') + format: csv + rows: | + id,b + 1,2 + 2,2 + expect: + format: csv + fixture: fake_fixture +""" + +test_my_model_duplicate_csv_yml = """ +unit_tests: + - name: test_missing_csv_file + model: my_model + given: + - input: ref('my_model_a') + format: csv + rows: | + id,a + 1,1 + - input: ref('my_model_b') + format: csv + rows: | + id,b + 1,2 + 2,2 + expect: + format: csv + fixture: test_my_model_basic_fixture +""" diff --git a/tests/functional/unit_testing/test_csv_fixtures.py b/tests/functional/unit_testing/test_csv_fixtures.py new file mode 100644 index 00000000000..2e10a395b83 --- /dev/null +++ b/tests/functional/unit_testing/test_csv_fixtures.py @@ -0,0 +1,223 @@ +import pytest +from dbt.exceptions import ParsingError, YamlParseDictError +from dbt.tests.util import run_dbt, write_file +from fixtures import ( + my_model_sql, + my_model_a_sql, + my_model_b_sql, + test_my_model_csv_yml, + datetime_test, + datetime_test_invalid_format_key, + datetime_test_invalid_csv_values, + test_my_model_file_csv_yml, + test_my_model_fixture_csv, + test_my_model_a_fixture_csv, + test_my_model_b_fixture_csv, + test_my_model_basic_fixture_csv, + test_my_model_a_numeric_fixture_csv, + test_my_model_a_empty_fixture_csv, + test_my_model_concat_fixture_csv, + test_my_model_mixed_csv_yml, + test_my_model_missing_csv_yml, + test_my_model_duplicate_csv_yml, +) + + +class TestUnitTestsWithInlineCSV: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "my_model_a.sql": my_model_a_sql, + "my_model_b.sql": my_model_b_sql, + "test_my_model.yml": test_my_model_csv_yml + datetime_test, + } + + def test_unit_test(self, project): + results = run_dbt(["run"]) + assert len(results) == 3 + + # Select by model name + results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) + assert len(results) == 5 + + # Check error with invalid format key + write_file( + test_my_model_csv_yml + datetime_test_invalid_format_key, + project.project_root, + "models", + "test_my_model.yml", + ) + with pytest.raises(YamlParseDictError): + results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) + + # Check error with csv format defined but dict on rows + write_file( + test_my_model_csv_yml + datetime_test_invalid_csv_values, + project.project_root, + "models", + "test_my_model.yml", + ) + with pytest.raises(ParsingError): + results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) + + +class TestUnitTestsWithFileCSV: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "my_model_a.sql": my_model_a_sql, + "my_model_b.sql": my_model_b_sql, + "test_my_model.yml": test_my_model_file_csv_yml + datetime_test, + } + + @pytest.fixture(scope="class") + def tests(self): + return { + "fixtures": { + "test_my_model_fixture.csv": test_my_model_fixture_csv, + "test_my_model_a_fixture.csv": test_my_model_a_fixture_csv, + "test_my_model_b_fixture.csv": test_my_model_b_fixture_csv, + "test_my_model_basic_fixture.csv": test_my_model_basic_fixture_csv, + "test_my_model_a_numeric_fixture.csv": test_my_model_a_numeric_fixture_csv, + "test_my_model_a_empty_fixture.csv": test_my_model_a_empty_fixture_csv, + "test_my_model_concat_fixture.csv": test_my_model_concat_fixture_csv, + } + } + + def test_unit_test(self, project): + results = run_dbt(["run"]) + assert len(results) == 3 + + # Select by model name + results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) + assert len(results) == 5 + + # Check error with invalid format key + write_file( + test_my_model_file_csv_yml + datetime_test_invalid_format_key, + project.project_root, + "models", + "test_my_model.yml", + ) + with pytest.raises(YamlParseDictError): + results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) + + # Check error with csv format defined but dict on rows + write_file( + test_my_model_file_csv_yml + datetime_test_invalid_csv_values, + project.project_root, + "models", + "test_my_model.yml", + ) + with pytest.raises(ParsingError): + results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) + + +class TestUnitTestsWithMixedCSV: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "my_model_a.sql": my_model_a_sql, + "my_model_b.sql": my_model_b_sql, + "test_my_model.yml": test_my_model_mixed_csv_yml + datetime_test, + } + + @pytest.fixture(scope="class") + def tests(self): + return { + "fixtures": { + "test_my_model_fixture.csv": test_my_model_fixture_csv, + "test_my_model_a_fixture.csv": test_my_model_a_fixture_csv, + "test_my_model_b_fixture.csv": test_my_model_b_fixture_csv, + "test_my_model_basic_fixture.csv": test_my_model_basic_fixture_csv, + "test_my_model_a_numeric_fixture.csv": test_my_model_a_numeric_fixture_csv, + "test_my_model_a_empty_fixture.csv": test_my_model_a_empty_fixture_csv, + "test_my_model_concat_fixture.csv": test_my_model_concat_fixture_csv, + } + } + + def test_unit_test(self, project): + results = run_dbt(["run"]) + assert len(results) == 3 + + # Select by model name + results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) + assert len(results) == 5 + + # Check error with invalid format key + write_file( + test_my_model_mixed_csv_yml + datetime_test_invalid_format_key, + project.project_root, + "models", + "test_my_model.yml", + ) + with pytest.raises(YamlParseDictError): + results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) + + # Check error with csv format defined but dict on rows + write_file( + test_my_model_mixed_csv_yml + datetime_test_invalid_csv_values, + project.project_root, + "models", + "test_my_model.yml", + ) + with pytest.raises(ParsingError): + results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) + + +class TestUnitTestsMissingCSVFile: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "my_model_a.sql": my_model_a_sql, + "my_model_b.sql": my_model_b_sql, + "test_my_model.yml": test_my_model_missing_csv_yml, + } + + def test_missing(self, project): + results = run_dbt(["run"]) + assert len(results) == 3 + + # Select by model name + expected_error = "Could not find fixture file fake_fixture for unit test" + with pytest.raises(ParsingError, match=expected_error): + results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) + + +class TestUnitTestsDuplicateCSVFile: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "my_model_a.sql": my_model_a_sql, + "my_model_b.sql": my_model_b_sql, + "test_my_model.yml": test_my_model_duplicate_csv_yml, + } + + @pytest.fixture(scope="class") + def tests(self): + return { + "fixtures": { + "one-folder": { + "test_my_model_basic_fixture.csv": test_my_model_basic_fixture_csv, + }, + "another-folder": { + "test_my_model_basic_fixture.csv": test_my_model_basic_fixture_csv, + }, + } + } + + def test_duplicate(self, project): + results = run_dbt(["run"]) + assert len(results) == 3 + + # Select by model name + with pytest.raises(ParsingError) as exc: + results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) + expected_error = "Found multiple fixture files named test_my_model_basic_fixture at ['one-folder/test_my_model_basic_fixture.csv', 'another-folder/test_my_model_basic_fixture.csv']" + # doing the match inline with the pytest.raises caused a bad character error with the dashes. So we do it here. + assert exc.match(expected_error) diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index 5c3055525d0..2a631c23efe 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -1,125 +1,23 @@ import pytest from dbt.tests.util import run_dbt, write_file, get_manifest, get_artifact -from dbt.exceptions import DuplicateResourceNameError, ParsingError, YamlParseDictError - -my_model_sql = """ -SELECT -a+b as c, -concat(string_a, string_b) as string_c, -not_testing, date_a, -{{ dbt.string_literal(type_numeric()) }} as macro_call, -{{ dbt.string_literal(var('my_test')) }} as var_call, -{{ dbt.string_literal(env_var('MY_TEST', 'default')) }} as env_var_call, -{{ dbt.string_literal(invocation_id) }} as invocation_id -FROM {{ ref('my_model_a')}} my_model_a -JOIN {{ ref('my_model_b' )}} my_model_b -ON my_model_a.id = my_model_b.id -""" - -my_model_a_sql = """ -SELECT -1 as a, -1 as id, -2 as not_testing, -'a' as string_a, -DATE '2020-01-02' as date_a -""" - -my_model_b_sql = """ -SELECT -2 as b, -1 as id, -2 as c, -'b' as string_b -""" - -test_my_model_yml = """ -unit_tests: - - name: test_my_model - model: my_model - given: - - input: ref('my_model_a') - rows: - - {id: 1, a: 1} - - input: ref('my_model_b') - rows: - - {id: 1, b: 2} - - {id: 2, b: 2} - expect: - rows: - - {c: 2} - - - name: test_my_model_empty - model: my_model - given: - - input: ref('my_model_a') - rows: [] - - input: ref('my_model_b') - rows: - - {id: 1, b: 2} - - {id: 2, b: 2} - expect: - rows: [] - - - name: test_my_model_overrides - model: my_model - given: - - input: ref('my_model_a') - rows: - - {id: 1, a: 1} - - input: ref('my_model_b') - rows: - - {id: 1, b: 2} - - {id: 2, b: 2} - overrides: - macros: - type_numeric: override - invocation_id: 123 - vars: - my_test: var_override - env_vars: - MY_TEST: env_var_override - expect: - rows: - - {macro_call: override, var_call: var_override, env_var_call: env_var_override, invocation_id: 123} - - - name: test_my_model_string_concat - model: my_model - given: - - input: ref('my_model_a') - rows: - - {id: 1, string_a: a} - - input: ref('my_model_b') - rows: - - {id: 1, string_b: b} - expect: - rows: - - {string_c: ab} - config: - tags: test_this -""" - -datetime_test = """ - - name: test_my_model_datetime - model: my_model - given: - - input: ref('my_model_a') - rows: - - {id: 1, date_a: "2020-01-01"} - - input: ref('my_model_b') - rows: - - {id: 1} - expect: - rows: - - {date_a: "2020-01-01"} -""" +from dbt.exceptions import DuplicateResourceNameError +from fixtures import ( + my_model_vars_sql, + my_model_a_sql, + my_model_b_sql, + test_my_model_yml, + datetime_test, + my_incremental_model_sql, + event_sql, + test_my_model_incremental_yml, +) class TestUnitTests: @pytest.fixture(scope="class") def models(self): return { - "my_model.sql": my_model_sql, + "my_model.sql": my_model_vars_sql, "my_model_a.sql": my_model_a_sql, "my_model_b.sql": my_model_b_sql, "test_my_model.yml": test_my_model_yml + datetime_test, @@ -186,220 +84,6 @@ def test_basic(self, project): run_dbt(["run", "--no-partial-parse", "--select", "my_model"]) -test_my_model_csv_yml = """ -unit_tests: - - name: test_my_model - model: my_model - given: - - input: ref('my_model_a') - format: csv - rows: | - id,a - 1,1 - - input: ref('my_model_b') - format: csv - rows: | - id,b - 1,2 - 2,2 - expect: - format: csv - rows: | - c - 2 - - - name: test_my_model_empty - model: my_model - given: - - input: ref('my_model_a') - rows: [] - - input: ref('my_model_b') - format: csv - rows: | - id,b - 1,2 - 2,2 - expect: - rows: [] - - name: test_my_model_overrides - model: my_model - given: - - input: ref('my_model_a') - format: csv - rows: | - id,a - 1,1 - - input: ref('my_model_b') - format: csv - rows: | - id,b - 1,2 - 2,2 - overrides: - macros: - type_numeric: override - invocation_id: 123 - vars: - my_test: var_override - env_vars: - MY_TEST: env_var_override - expect: - rows: - - {macro_call: override, var_call: var_override, env_var_call: env_var_override, invocation_id: 123} - - name: test_my_model_string_concat - model: my_model - given: - - input: ref('my_model_a') - format: csv - rows: | - id,string_a - 1,a - - input: ref('my_model_b') - format: csv - rows: | - id,string_b - 1,b - expect: - format: csv - rows: | - string_c - ab - config: - tags: test_this -""" - -datetime_test_invalid_format = """ - - name: test_my_model_datetime - model: my_model - given: - - input: ref('my_model_a') - format: xxxx - rows: - - {id: 1, date_a: "2020-01-01"} - - input: ref('my_model_b') - rows: - - {id: 1} - expect: - rows: - - {date_a: "2020-01-01"} -""" - -datetime_test_invalid_format2 = """ - - name: test_my_model_datetime - model: my_model - given: - - input: ref('my_model_a') - format: csv - rows: - - {id: 1, date_a: "2020-01-01"} - - input: ref('my_model_b') - rows: - - {id: 1} - expect: - rows: - - {date_a: "2020-01-01"} -""" - - -class TestUnitTestsWithInlineCSV: - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "my_model_a.sql": my_model_a_sql, - "my_model_b.sql": my_model_b_sql, - "test_my_model.yml": test_my_model_csv_yml + datetime_test, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return {"vars": {"my_test": "my_test_var"}} - - def test_basic(self, project): - results = run_dbt(["run"]) - assert len(results) == 3 - - # Select by model name - results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) - assert len(results) == 5 - - # Check error with invalid format - write_file( - test_my_model_csv_yml + datetime_test_invalid_format, - project.project_root, - "models", - "test_my_model.yml", - ) - with pytest.raises(YamlParseDictError): - results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) - - # Check error with format not matching rows - write_file( - test_my_model_csv_yml + datetime_test_invalid_format2, - project.project_root, - "models", - "test_my_model.yml", - ) - with pytest.raises(ParsingError): - results = run_dbt(["unit-test", "--select", "my_model"], expect_pass=False) - - -event_sql = """ -select DATE '2020-01-01' as event_time, 1 as event -union all -select DATE '2020-01-02' as event_time, 2 as event -union all -select DATE '2020-01-03' as event_time, 3 as event -""" - -my_incremental_model_sql = """ -{{ - config( - materialized='incremental' - ) -}} - -select * from {{ ref('events') }} -{% if is_incremental() %} -where event_time > (select max(event_time) from {{ this }}) -{% endif %} -""" - -test_my_model_incremental_yml = """ -unit_tests: - - name: incremental_false - model: my_incremental_model - overrides: - macros: - is_incremental: false - given: - - input: ref('events') - rows: - - {event_time: "2020-01-01", event: 1} - expect: - rows: - - {event_time: "2020-01-01", event: 1} - - name: incremental_true - model: my_incremental_model - overrides: - macros: - is_incremental: true - given: - - input: ref('events') - rows: - - {event_time: "2020-01-01", event: 1} - - {event_time: "2020-01-02", event: 2} - - {event_time: "2020-01-03", event: 3} - - input: this - rows: - - {event_time: "2020-01-01", event: 1} - expect: - rows: - - {event_time: "2020-01-02", event: 2} - - {event_time: "2020-01-03", event: 3} -""" - - class TestUnitTestIncrementalModel: @pytest.fixture(scope="class") def models(self):