From d246bab0eb656eae249f8b37c750dffdad5fc3eb Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 25 Sep 2019 16:27:44 -0600 Subject: [PATCH] Fix seeds in dependencies Modified a test to cover the bug Update the FilePath definition Add a new attribute to seeds (seed_file_path) Test fixes Added --no-version-check flag to dbt seed --- core/dbt/clients/system.py | 11 ++- core/dbt/context/common.py | 10 ++- core/dbt/contracts/graph/compiled.py | 10 ++- core/dbt/contracts/graph/manifest.py | 25 ++++-- core/dbt/contracts/graph/parsed.py | 7 ++ core/dbt/main.py | 4 +- core/dbt/parser/hooks.py | 12 +-- core/dbt/parser/search.py | 14 ++- core/dbt/parser/seeds.py | 16 ++++ .../local_dependency/data/seed.csv | 2 + .../models/model_to_import.sql | 4 +- .../test_local_dependency.py | 15 +++- .../test_docs_generate.py | 90 ++++++++++--------- test/unit/test_docs_blocks.py | 2 +- test/unit/test_graph.py | 3 +- test/unit/test_loader.py | 3 +- test/unit/test_manifest.py | 3 +- test/unit/test_parser.py | 22 +++-- 18 files changed, 160 insertions(+), 93 deletions(-) create mode 100644 test/integration/006_simple_dependency_test/local_dependency/data/seed.csv diff --git a/core/dbt/clients/system.py b/core/dbt/clients/system.py index 9023ed85299..081d78a7108 100644 --- a/core/dbt/clients/system.py +++ b/core/dbt/clients/system.py @@ -45,13 +45,16 @@ def find_matching(root_path, for local_file in local_files: absolute_path = os.path.join(current_path, local_file) relative_path = os.path.relpath( - absolute_path, absolute_path_to_search) + absolute_path, absolute_path_to_search + ) if fnmatch.fnmatch(local_file, file_pattern): matching.append({ - 'searched_path': relative_path_to_search, - 'absolute_path': absolute_path, - 'relative_path': relative_path, + 'searched_path': os.path.normcase( + relative_path_to_search + ), + 'absolute_path': os.path.normcase(absolute_path), + 'relative_path': os.path.normcase(relative_path), }) return matching diff --git a/core/dbt/context/common.py b/core/dbt/context/common.py index efc5b8d9f5e..6e3ad663455 100644 --- a/core/dbt/context/common.py +++ b/core/dbt/context/common.py @@ -1,5 +1,7 @@ +import agate import json import os +from typing import Union, Callable from dbt.adapters.factory import get_adapter from dbt.node_types import NodeType @@ -8,6 +10,8 @@ import dbt.clients.jinja import dbt.clients.agate_helper +from dbt.contracts.graph.compiled import CompiledSeedNode +from dbt.contracts.graph.parsed import ParsedSeedNode import dbt.exceptions import dbt.flags import dbt.tracking @@ -353,9 +357,11 @@ def generate_config_context(cli_vars): return _add_tracking(context) -def _build_load_agate_table(model): +def _build_load_agate_table( + model: Union[ParsedSeedNode, CompiledSeedNode] +) -> Callable[[], agate.Table]: def load_agate_table(): - path = model.original_file_path + path = model.seed_file_path try: table = dbt.clients.agate_helper.from_csv(path) except ValueError as e: diff --git a/core/dbt/contracts/graph/compiled.py b/core/dbt/contracts/graph/compiled.py index f840137745d..ae1413a3dbd 100644 --- a/core/dbt/contracts/graph/compiled.py +++ b/core/dbt/contracts/graph/compiled.py @@ -13,6 +13,7 @@ ) from dbt.node_types import NodeType from dbt.contracts.util import Replaceable +from dbt.exceptions import InternalException from hologram import JsonSchemaMixin from dataclasses import dataclass, field @@ -87,6 +88,13 @@ class CompiledRPCNode(CompiledNode): @dataclass class CompiledSeedNode(CompiledNode): resource_type: NodeType = field(metadata={'restrict': [NodeType.Seed]}) + seed_file_path: str = '' + + def __post_init__(self): + if self.seed_file_path == '': + raise InternalException( + 'Seeds should always have a seed_file_path' + ) @property def empty(self): @@ -191,7 +199,7 @@ def parsed_instance_for(compiled: CompiledNode) -> ParsedNode: raise ValueError('invalid resource_type: {}' .format(compiled.resource_type)) - # validate=False to allow extra keys from copmiling + # validate=False to allow extra keys from compiling return cls.from_dict(compiled.to_dict(), validate=False) diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 43ac5663d1f..c9f880c407c 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -25,16 +25,31 @@ class FilePath(JsonSchemaMixin): searched_path: str relative_path: str - absolute_path: str + project_root: str @property - def search_key(self): - # TODO: should this be project root + original_file_path? + def search_key(self) -> str: + # TODO: should this be project name + path relative to project root? return self.absolute_path @property - def original_file_path(self): - return os.path.join(self.searched_path, self.relative_path) + def full_path(self) -> str: + # useful for symlink preservation + return os.path.normcase(os.path.join( + self.project_root, self.searched_path, self.relative_path + )) + + @property + def absolute_path(self) -> str: + return os.path.normcase(os.path.abspath(self.full_path)) + + @property + def original_file_path(self) -> str: + # this is mostly used for reporting errors. It doesn't show the project + # name, should it? + return os.path.join( + self.searched_path, self.relative_path + ) @dataclass diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index 14cf83aae9a..3dc82c43b1a 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -263,6 +263,13 @@ class ParsedRPCNode(ParsedNode): @dataclass class ParsedSeedNode(ParsedNode): resource_type: NodeType = field(metadata={'restrict': [NodeType.Seed]}) + seed_file_path: str = '' + + def __post_init__(self): + if self.seed_file_path == '': + raise dbt.exceptions.InternalException( + 'Seeds should always have a seed_file_path' + ) @property def empty(self): diff --git a/core/dbt/main.py b/core/dbt/main.py index 1e689f2a7ed..c44b2dc9b70 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -823,16 +823,16 @@ def parse_args(args): compile_sub = _build_compile_subparser(subs, base_subparser) generate_sub = _build_docs_generate_subparser(docs_subs, base_subparser) test_sub = _build_test_subparser(subs, base_subparser) + seed_sub = _build_seed_subparser(subs, base_subparser) # --threads, --no-version-check _add_common_arguments(run_sub, compile_sub, generate_sub, test_sub, - rpc_sub) + rpc_sub, seed_sub) # --models, --exclude _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub) _add_selection_arguments(snapshot_sub, models_name='select') # --full-refresh _add_table_mutability_arguments(run_sub, compile_sub) - _build_seed_subparser(subs, base_subparser) _build_docs_serve_subparser(docs_subs, base_subparser) _build_source_snapshot_freshness_subparser(source_subs, base_subparser) _build_run_operation_subparser(subs, base_subparser) diff --git a/core/dbt/parser/hooks.py b/core/dbt/parser/hooks.py index d2c4e3a6199..b724c02dffe 100644 --- a/core/dbt/parser/hooks.py +++ b/core/dbt/parser/hooks.py @@ -1,4 +1,3 @@ -import os from dataclasses import dataclass from typing import Iterable, Iterator, Union, List, Tuple @@ -71,16 +70,11 @@ class HookParser(SimpleParser[HookBlock, ParsedHookNode]): def transform(self, node): return node - def get_paths(self): - searched_path = '.' - relative_path = 'dbt_project.yml' - absolute_path = os.path.normcase(os.path.abspath(os.path.join( - self.project.project_root, searched_path, relative_path - ))) + def get_paths(self) -> List[FilePath]: path = FilePath( + project_root=self.project.project_root, searched_path='.', - relative_path='relative_path', - absolute_path=absolute_path, + relative_path='dbt_project.yml', ) return [path] diff --git a/core/dbt/parser/search.py b/core/dbt/parser/search.py index ec165016e25..b343cc98189 100644 --- a/core/dbt/parser/search.py +++ b/core/dbt/parser/search.py @@ -8,7 +8,7 @@ from dbt.clients.system import find_matching from dbt.config import Project from dbt.contracts.graph.manifest import SourceFile, FilePath -from dbt.exceptions import CompilationException +from dbt.exceptions import CompilationException, InternalException @dataclass @@ -70,9 +70,15 @@ def __iter__(self) -> Iterator[FilePath]: root = self.project.project_root for result in find_matching(root, self.relative_dirs, ext): - file_match = FilePath(**{ - k: os.path.normcase(v) for k, v in result.items() - }) + if 'searched_path' not in result or 'relative_path' not in result: + raise InternalException( + 'Invalid result from find_matching: {}'.format(result) + ) + file_match = FilePath( + searched_path=result['searched_path'], + relative_path=result['relative_path'], + project_root=root, + ) yield file_match diff --git a/core/dbt/parser/seeds.py b/core/dbt/parser/seeds.py index 83027584bd5..4b126fc9947 100644 --- a/core/dbt/parser/seeds.py +++ b/core/dbt/parser/seeds.py @@ -30,3 +30,19 @@ def render_with_context( def load_file(self, match: FilePath) -> SourceFile: return SourceFile.seed(match) + + def _create_parsetime_node( + self, + block: FileBlock, + path: str, + config: SourceConfig, + name=None, + **kwargs, + ) -> ParsedSeedNode: + return super()._create_parsetime_node( + block=block, + path=path, + config=config, + name=name, + seed_file_path=block.path.full_path, + ) diff --git a/test/integration/006_simple_dependency_test/local_dependency/data/seed.csv b/test/integration/006_simple_dependency_test/local_dependency/data/seed.csv new file mode 100644 index 00000000000..3ff3deb87eb --- /dev/null +++ b/test/integration/006_simple_dependency_test/local_dependency/data/seed.csv @@ -0,0 +1,2 @@ +id +1 diff --git a/test/integration/006_simple_dependency_test/local_dependency/models/model_to_import.sql b/test/integration/006_simple_dependency_test/local_dependency/models/model_to_import.sql index ae5662470ed..4b91aa0f2fa 100644 --- a/test/integration/006_simple_dependency_test/local_dependency/models/model_to_import.sql +++ b/test/integration/006_simple_dependency_test/local_dependency/models/model_to_import.sql @@ -1,3 +1 @@ - - -select 1 as id +select * from {{ ref('seed') }} diff --git a/test/integration/006_simple_dependency_test/test_local_dependency.py b/test/integration/006_simple_dependency_test/test_local_dependency.py index 592b71f532f..a1133bac293 100644 --- a/test/integration/006_simple_dependency_test/test_local_dependency.py +++ b/test/integration/006_simple_dependency_test/test_local_dependency.py @@ -37,11 +37,11 @@ class TestSimpleDependency(BaseDependencyTest): @property def schema(self): - return "local_dependency_006" + return 'local_dependency_006' @property def models(self): - return "local_models" + return 'local_models' def base_schema(self): return self.unique_schema() @@ -51,8 +51,9 @@ def configured_schema(self): @use_profile('postgres') def test_postgres_local_dependency(self): - self.run_dbt(["deps"]) - results = self.run_dbt(["run"]) + self.run_dbt(['deps']) + self.run_dbt(['seed']) + results = self.run_dbt(['run']) self.assertEqual(len(results), 3) self.assertEqual({r.node.schema for r in results}, {self.base_schema(), self.configured_schema()}) @@ -103,6 +104,11 @@ def configured_schema(self): def test_postgres_local_dependency_out_of_date(self, mock_get): mock_get.return_value = dbt.semver.VersionSpecifier.from_version_string('0.0.1') self.run_dbt(['deps']) + # check seed + with self.assertRaises(dbt.exceptions.DbtProjectError) as exc: + self.run_dbt(['seed']) + self.assertIn('--no-version-check', str(exc.exception)) + # check run too with self.assertRaises(dbt.exceptions.DbtProjectError) as exc: self.run_dbt(['run']) self.assertIn('--no-version-check', str(exc.exception)) @@ -112,6 +118,7 @@ def test_postgres_local_dependency_out_of_date(self, mock_get): def test_postgres_local_dependency_out_of_date_no_check(self, mock_get): mock_get.return_value = dbt.semver.VersionSpecifier.from_version_string('0.0.1') self.run_dbt(['deps']) + self.run_dbt(['seed', '--no-version-check']) results = self.run_dbt(['run', '--no-version-check']) self.assertEqual(len(results), 3) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 69107fcf221..2300e20f689 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1,7 +1,7 @@ import hashlib import json import os -from datetime import datetime, timedelta +from datetime import datetime from unittest.mock import ANY, patch from test.integration.base import DBTIntegrationTest, use_profile, AnyFloat, \ @@ -823,7 +823,7 @@ def expected_seeded_manifest(self, model_database=None): 'model.test.model': { 'build_path': None, 'name': 'model', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'resource_type': 'model', 'path': 'model.sql', 'original_file_path': model_sql_path, @@ -881,7 +881,8 @@ def expected_seeded_manifest(self, model_database=None): 'patch_path': None, 'path': 'seed.csv', 'name': 'seed', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, + 'seed_file_path': normalize(os.path.join(self.test_root_dir, 'seed', 'seed.csv')), 'resource_type': 'seed', 'raw_sql': '', 'package_name': 'test', @@ -929,7 +930,7 @@ def expected_seeded_manifest(self, model_database=None): 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(model=ref('model'), column_name='id') }}", 'refs': [['model']], 'resource_type': 'test', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': ['schema'], @@ -965,7 +966,7 @@ def expected_seeded_manifest(self, model_database=None): 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(model=ref('model'), ) }}", 'refs': [['model']], 'resource_type': 'test', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': ['schema'], @@ -1001,7 +1002,7 @@ def expected_seeded_manifest(self, model_database=None): 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(model=ref('model'), column_name='id') }}", 'refs': [['model']], 'resource_type': 'test', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': ['schema'], @@ -1059,7 +1060,7 @@ def expected_seeded_manifest(self, model_database=None): 'path': self._path_to('seed', 'seed.csv'), 'checksum': { 'name': 'path', - 'checksum': self._path_to('seed', 'seed.csv')['absolute_path'], + 'checksum': self._absolute_path_to('seed', 'seed.csv'), }, 'docs': [], 'macros': [], @@ -1149,7 +1150,7 @@ def expected_postgres_references_manifest(self, model_database=None): ), 'refs': [], 'resource_type': 'model', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], @@ -1216,7 +1217,7 @@ def expected_postgres_references_manifest(self, model_database=None): ), 'refs': [['ephemeral_copy']], 'resource_type': 'model', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], @@ -1281,7 +1282,7 @@ def expected_postgres_references_manifest(self, model_database=None): ), 'refs': [['ephemeral_summary']], 'resource_type': 'model', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'sources': [], 'tags': [], @@ -1315,7 +1316,8 @@ def expected_postgres_references_manifest(self, model_database=None): 'raw_sql': '', 'refs': [], 'resource_type': 'seed', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, + 'seed_file_path': normalize(os.path.join(self.test_root_dir, 'seed', 'seed.csv')), 'schema': my_schema_name, 'database': self.default_database, 'tags': [], @@ -1361,7 +1363,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'package_name': 'test', 'path': self.dir('ref_models/schema.yml'), 'resource_type': 'source', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'source_description': 'My source', 'source_name': 'my_source', @@ -1378,7 +1380,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'unique_id': 'test.column_info', }, 'test.ephemeral_summary': { @@ -1390,7 +1392,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'unique_id': 'test.ephemeral_summary', }, 'test.source_info': { @@ -1400,7 +1402,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'unique_id': 'test.source_info', }, 'test.summary_count': { @@ -1410,7 +1412,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'unique_id': 'test.summary_count', }, 'test.summary_first_name': { @@ -1420,7 +1422,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'unique_id': 'test.summary_first_name', }, 'test.table_info': { @@ -1430,7 +1432,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'unique_id': 'test.table_info', }, 'test.view_summary': { @@ -1443,7 +1445,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'unique_id': 'test.view_summary', }, }, @@ -1507,7 +1509,7 @@ def expected_postgres_references_manifest(self, model_database=None): normalize('seed/seed.csv'): { 'checksum': { 'name': 'path', - 'checksum': self._path_to('seed', 'seed.csv')['absolute_path'], + 'checksum': self._absolute_path_to('seed', 'seed.csv'), }, 'docs': [], 'macros': [], @@ -1581,7 +1583,7 @@ def expected_bigquery_complex_manifest(self): 'raw_sql': LineIndifferent(_read_file(clustered_sql_path).rstrip('\r\n')), 'refs': [['seed']], 'resource_type': 'model', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], @@ -1638,7 +1640,7 @@ def expected_bigquery_complex_manifest(self): 'raw_sql': LineIndifferent(_read_file(multi_clustered_sql_path).rstrip('\r\n')), 'refs': [['seed']], 'resource_type': 'model', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], @@ -1696,7 +1698,7 @@ def expected_bigquery_complex_manifest(self): 'raw_sql': LineIndifferent(_read_file(nested_view_sql_path).rstrip('\r\n')), 'refs': [['nested_table']], 'resource_type': 'model', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], @@ -1755,7 +1757,7 @@ def expected_bigquery_complex_manifest(self): 'raw_sql': LineIndifferent(_read_file(nested_table_sql_path).rstrip('\r\n')), 'refs': [], 'resource_type': 'model', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], @@ -1769,7 +1771,8 @@ def expected_bigquery_complex_manifest(self): 'patch_path': None, 'path': 'seed.csv', 'name': 'seed', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, + 'seed_file_path': normalize(os.path.join(self.test_root_dir, 'seed', 'seed.csv')), 'resource_type': 'seed', 'raw_sql': '', 'package_name': 'test', @@ -1874,7 +1877,7 @@ def expected_bigquery_complex_manifest(self): normalize('seed/seed.csv'): { 'checksum': { 'name': 'path', - 'checksum': self._path_to('seed', 'seed.csv')['absolute_path'], + 'checksum': self._absolute_path_to('seed', 'seed.csv'), }, 'path': self._path_to('seed', 'seed.csv'), 'nodes': ['seed.test.seed'], @@ -1907,17 +1910,19 @@ def _checksum_file(self, path): } def _path_to(self, searched_path: str, relative_path: str): - if searched_path == '.': - absolute_path = os.path.join(self.test_root_dir, relative_path) - else: - absolute_path = os.path.join(self.test_root_dir, searched_path, relative_path) - return { 'searched_path': normalize(searched_path), 'relative_path': normalize(relative_path), - 'absolute_path': normalize(absolute_path), + 'project_root': normalize(self.test_root_dir), } + def _absolute_path_to(self, searched_path: str, relative_path: str): + return os.path.join( + normalize(self.test_root_dir), + normalize(searched_path), + normalize(relative_path) + ) + def expected_redshift_incremental_view_manifest(self): model_sql_path = self.dir('rs_models/model.sql') my_schema_name = self.unique_schema() @@ -1990,6 +1995,7 @@ def expected_redshift_incremental_view_manifest(self): 'path': 'seed.csv', 'name': 'seed', 'root_path': self.test_root_dir, + 'seed_file_path': normalize(os.path.join(self.test_root_dir, 'seed', 'seed.csv')), 'resource_type': 'seed', 'raw_sql': '', 'package_name': 'test', @@ -2061,7 +2067,7 @@ def expected_redshift_incremental_view_manifest(self): normalize('seed/seed.csv'): { 'checksum': { 'name': 'path', - 'checksum': self._path_to('seed', 'seed.csv')['absolute_path'], + 'checksum': self._absolute_path_to('seed', 'seed.csv'), }, 'path': self._path_to('seed', 'seed.csv'), 'docs': [], @@ -2208,7 +2214,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'raw_sql': LineIndifferent(_read_file(model_sql_path).rstrip('\r\n')), 'refs': [['seed']], 'resource_type': 'model', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': schema, 'database': model_database, 'tags': [], @@ -2258,7 +2264,8 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'raw_sql': '', 'refs': [], 'resource_type': 'seed', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, + 'seed_file_path': normalize(os.path.join(self.test_root_dir, 'seed', 'seed.csv')), 'schema': schema, 'database': self.default_database, 'tags': [], @@ -2310,7 +2317,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(model=ref('model'), column_name='id') }}", 'refs': [['model']], 'resource_type': 'test', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': schema, 'database': self.default_database, 'tags': ['schema'], @@ -2362,7 +2369,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(model=ref('model'), ) }}", 'refs': [['model']], 'resource_type': 'test', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': schema, 'sources': [], 'tags': ['schema'], @@ -2414,7 +2421,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(model=ref('model'), column_name='id') }}", 'refs': [['model']], 'resource_type': 'test', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': schema, 'sources': [], 'tags': ['schema'], @@ -2530,7 +2537,7 @@ def expected_postgres_references_run_results(self): ), 'refs': [['ephemeral_copy']], 'resource_type': 'model', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], @@ -2618,7 +2625,7 @@ def expected_postgres_references_run_results(self): ), 'refs': [['ephemeral_summary']], 'resource_type': 'model', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], @@ -2668,7 +2675,8 @@ def expected_postgres_references_run_results(self): 'raw_sql': '', 'refs': [], 'resource_type': 'seed', - 'root_path': OneOf(self.test_root_dir, self.initial_dir), + 'root_path': self.test_root_dir, + 'seed_file_path': normalize(os.path.join(self.test_root_dir, 'seed', 'seed.csv')), 'schema': my_schema_name, 'database': self.default_database, 'tags': [], diff --git a/test/unit/test_docs_blocks.py b/test/unit/test_docs_blocks.py index 0676683e7fc..0ea096cce0f 100644 --- a/test/unit/test_docs_blocks.py +++ b/test/unit/test_docs_blocks.py @@ -113,7 +113,7 @@ def setUp(self): def _build_file(self, contents, relative_path) -> FileBlock: match = FilePath( relative_path=relative_path, - absolute_path=self.testfile_path, + project_root=self.root_path, searched_path=self.subdir_path, ) source_file = SourceFile(path=match, checksum=FileHash.empty()) diff --git a/test/unit/test_graph.py b/test/unit/test_graph.py index fb9b0ef001d..519c150cd2c 100644 --- a/test/unit/test_graph.py +++ b/test/unit/test_graph.py @@ -132,7 +132,7 @@ def use_models(self, models): for k, v in models.items(): path = FilePath( searched_path='models', - absolute_path=os.path.normcase(os.path.abspath('models/{}.sql'.format(k))), + project_root=os.path.normcase(os.getcwd()), relative_path='{}.sql'.format(k), ) source_file = SourceFile(path=path, checksum=FileHash.empty()) @@ -144,7 +144,6 @@ def load_manifest(self, config): loader.load() return loader.create_manifest() - def test__single_model(self): self.use_models({ 'model_one': 'select * from events', diff --git a/test/unit/test_loader.py b/test/unit/test_loader.py index 9324694f2bd..079dde1c62f 100644 --- a/test/unit/test_loader.py +++ b/test/unit/test_loader.py @@ -1,6 +1,5 @@ import unittest from unittest import mock -from os.path import join as pjoin from .utils import config_from_parts_or_dicts, normalize @@ -82,7 +81,7 @@ def _new_file(self, searched, name, match): path = FilePath( searched_path=normalize(searched), relative_path=normalize(name), - absolute_path=normalize(pjoin(self.root_project_config.project_root, searched, name)), + project_root=normalize(self.root_project_config.project_root), ) return SourceFile(path=path, checksum=checksum) diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index 3074b787807..218e410eb66 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -341,7 +341,8 @@ def test_get_resource_fqns(self): path='seed.csv', original_file_path='seed.csv', root_path='', - raw_sql='-- csv --' + raw_sql='-- csv --', + seed_file_path='data/seed.csv' ) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index a2dd9708856..8d394ab3214 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -30,7 +30,7 @@ def get_abs_os_path(unix_path): - return os.path.abspath(normalize(unix_path)) + return normalize(os.path.abspath(unix_path)) class BaseParserTest(unittest.TestCase): @@ -100,9 +100,7 @@ def file_block_for(self, data: str, filename: str, searched: str): path = FilePath( searched_path=searched, relative_path=filename, - absolute_path=os.path.normpath(os.path.abspath( - os.path.join(root_dir, searched, filename) - )), + project_root=root_dir, ) source_file = SourceFile( path=path, @@ -260,7 +258,7 @@ def test__parse_basic_source_tests(self): self.assertEqual(tests[1].column_name, 'color') self.assertEqual(tests[1].fqn, ['snowplow', 'schema_test', tests[1].name]) - path = os.path.abspath('./dbt_modules/snowplow/models/test_one.yml') + path = get_abs_os_path('./dbt_modules/snowplow/models/test_one.yml') self.assertIn(path, self.parser.results.files) self.assertEqual(sorted(self.parser.results.files[path].nodes), [t.unique_id for t in tests]) @@ -329,7 +327,7 @@ def test__parse_basic_model_tests(self): self.assertEqual(tests[2].fqn, ['snowplow', 'schema_test', tests[2].name]) self.assertEqual(tests[2].unique_id.split('.'), ['test', 'snowplow', tests[2].name]) - path = os.path.abspath('./dbt_modules/snowplow/models/test_one.yml') + path = get_abs_os_path('./dbt_modules/snowplow/models/test_one.yml') self.assertIn(path, self.parser.results.files) self.assertEqual(sorted(self.parser.results.files[path].nodes), [t.unique_id for t in tests]) @@ -372,7 +370,7 @@ def test_basic(self): raw_sql=raw_sql, ) self.assertEqual(node, expected) - path = os.path.abspath('./dbt_modules/snowplow/models/nested/model_1.sql') + path = get_abs_os_path('./dbt_modules/snowplow/models/nested/model_1.sql') self.assertIn(path, self.parser.results.files) self.assertEqual(self.parser.results.files[path].nodes, ['model.snowplow.model_1']) @@ -440,7 +438,7 @@ def test_single_block(self): raw_sql=raw_sql, ) self.assertEqual(node, expected) - path = os.path.abspath('./dbt_modules/snowplow/snapshots/nested/snap_1.sql') + path = get_abs_os_path('./dbt_modules/snowplow/snapshots/nested/snap_1.sql') self.assertIn(path, self.parser.results.files) self.assertEqual(self.parser.results.files[path].nodes, ['snapshot.snowplow.foo']) @@ -515,7 +513,7 @@ def test_multi_block(self): ) self.assertEqual(nodes[0], expect_bar) self.assertEqual(nodes[1], expect_foo) - path = os.path.abspath('./dbt_modules/snowplow/snapshots/nested/snap_1.sql') + path = get_abs_os_path('./dbt_modules/snowplow/snapshots/nested/snap_1.sql') self.assertIn(path, self.parser.results.files) self.assertEqual(sorted(self.parser.results.files[path].nodes), ['snapshot.snowplow.bar', 'snapshot.snowplow.foo']) @@ -549,7 +547,7 @@ def test_single_block(self): raw_sql=raw_sql ) self.assertEqual(macro, expected) - path = os.path.abspath('./dbt_modules/snowplow/macros/macro.sql') + path = get_abs_os_path('./dbt_modules/snowplow/macros/macro.sql') self.assertIn(path, self.parser.results.files) self.assertEqual(self.parser.results.files[path].macros, ['macro.snowplow.foo']) @@ -591,7 +589,7 @@ def test_basic(self): raw_sql=raw_sql, ) self.assertEqual(node, expected) - path = os.path.abspath('./dbt_modules/snowplow/tests/test_1.sql') + path = get_abs_os_path('./dbt_modules/snowplow/tests/test_1.sql') self.assertIn(path, self.parser.results.files) self.assertEqual(self.parser.results.files[path].nodes, ['test.snowplow.test_1']) @@ -632,7 +630,7 @@ def test_basic(self): raw_sql=raw_sql, ) self.assertEqual(node, expected) - path = os.path.abspath('./dbt_modules/snowplow/analyses/nested/analysis_1.sql') + path = get_abs_os_path('./dbt_modules/snowplow/analyses/nested/analysis_1.sql') self.assertIn(path, self.parser.results.files) self.assertEqual(self.parser.results.files[path].nodes, ['analysis.snowplow.analysis_1'])