diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 722eac66d58..1557bc8e4d0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -77,7 +77,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8] # TODO: support unit testing for python 3.9 (https://github.com/dbt-labs/dbt/issues/3689) + python-version: [3.7, 3.8, 3.9] env: TOXENV: "unit" diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ff3dea4799..d9f2741a2e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Under the hood Add --indirect-selection parameter to profiles.yml and builtin DBT_ env vars; stringified parameter to enable multi-modal use ([#3997](https://github.com/dbt-labs/dbt-core/issues/3997), [PR #4270](https://github.com/dbt-labs/dbt-core/pull/4270)) +- Fix filesystem searcher test failure on Python 3.9 ([#3689](https://github.com/dbt-labs/dbt-core/issues/3689), [#4271](https://github.com/dbt-labs/dbt-core/pull/4271)) ## dbt-core 1.0.0rc1 (November 10, 2021) diff --git a/core/dbt/parser/macros.py b/core/dbt/parser/macros.py index cd928fab0a3..5b5d8f21a91 100644 --- a/core/dbt/parser/macros.py +++ b/core/dbt/parser/macros.py @@ -11,7 +11,7 @@ from dbt.events.types import MacroFileParse from dbt.node_types import NodeType from dbt.parser.base import BaseParser -from dbt.parser.search import FileBlock, FilesystemSearcher +from dbt.parser.search import FileBlock, filesystem_search from dbt.utils import MACRO_PREFIX @@ -19,11 +19,11 @@ class MacroParser(BaseParser[ParsedMacro]): # This is only used when creating a MacroManifest separate # from the normal parsing flow. def get_paths(self) -> List[FilePath]: - return list(FilesystemSearcher( + return filesystem_search( project=self.project, relative_dirs=self.project.macro_paths, extension='.sql', - )) + ) @property def resource_type(self) -> NodeType: diff --git a/core/dbt/parser/read_files.py b/core/dbt/parser/read_files.py index 7b0c7de8afb..1b721c9599b 100644 --- a/core/dbt/parser/read_files.py +++ b/core/dbt/parser/read_files.py @@ -6,7 +6,7 @@ from dbt.parser.schemas import yaml_from_file, schema_file_keys, check_format_version from dbt.exceptions import CompilationException -from dbt.parser.search import FilesystemSearcher +from dbt.parser.search import filesystem_search from typing import Optional @@ -86,9 +86,9 @@ def load_seed_source_file(match: FilePath, project_name) -> SourceFile: # them into a bunch of FileSource objects def get_source_files(project, paths, extension, parse_file_type, saved_files): # file path list - fp_list = list(FilesystemSearcher( + fp_list = filesystem_search( project, paths, extension - )) + ) # file block list fb_list = [] for fp in fp_list: diff --git a/core/dbt/parser/search.py b/core/dbt/parser/search.py index 15a72f2517c..0d55f581742 100644 --- a/core/dbt/parser/search.py +++ b/core/dbt/parser/search.py @@ -63,31 +63,24 @@ def contents(self): return self.block.full_block -class FilesystemSearcher(Iterable[FilePath]): - def __init__( - self, project: Project, relative_dirs: List[str], extension: str - ) -> None: - self.project = project - self.relative_dirs = relative_dirs - self.extension = extension - - def __iter__(self) -> Iterator[FilePath]: - ext = "[!.#~]*" + self.extension - - root = self.project.project_root - - for result in find_matching(root, self.relative_dirs, ext): - if 'searched_path' not in result or 'relative_path' not in result: - raise InternalException( - 'Invalid result from find_matching: {}'.format(result) - ) - file_match = FilePath( - searched_path=result['searched_path'], - relative_path=result['relative_path'], - modification_time=result['modification_time'], - project_root=root, +def filesystem_search(project: Project, relative_dirs: List[str], extension: str): + ext = "[!.#~]*" + extension + root = project.project_root + file_path_list = [] + for result in find_matching(root, relative_dirs, ext): + if 'searched_path' not in result or 'relative_path' not in result: + raise InternalException( + 'Invalid result from find_matching: {}'.format(result) ) - yield file_match + file_match = FilePath( + searched_path=result['searched_path'], + relative_path=result['relative_path'], + modification_time=result['modification_time'], + project_root=root, + ) + file_path_list.append(file_match) + + return file_path_list Block = Union[BlockContents, FullBlock] diff --git a/test/unit/test_graph.py b/test/unit/test_graph.py index c68a7e68a25..a9d594075ce 100644 --- a/test/unit/test_graph.py +++ b/test/unit/test_graph.py @@ -29,8 +29,7 @@ class GraphTest(unittest.TestCase): def tearDown(self): self.write_gpickle_patcher.stop() - self.file_system_patcher.stop() - self.mock_filesystem_constructor.stop() + self.mock_filesystem_search.stop() self.mock_hook_constructor.stop() self.load_state_check.stop() self.load_source_file_patcher.stop() @@ -66,22 +65,16 @@ def mock_write_gpickle(graph, outfile): self.mock_write_gpickle = self.write_gpickle_patcher.start() self.mock_write_gpickle.side_effect = mock_write_gpickle - # Create file system patcher and filesystem searcher - self.file_system_patcher = patch.object( - dbt.parser.search.FilesystemSearcher, '__new__' - ) - self.mock_filesystem_constructor = self.file_system_patcher.start() - def filesystem_iter(iter_self): - if 'sql' not in iter_self.extension: + # Create file filesystem searcher + self.filesystem_search = patch('dbt.parser.read_files.filesystem_search') + def mock_filesystem_search(project, relative_dirs, extension): + if 'sql' not in extension: return [] - if 'models' not in iter_self.relative_dirs: + if 'models' not in relative_dirs: return [] return [model.path for model in self.mock_models] - def create_filesystem_searcher(cls, project, relative_dirs, extension): - result = MagicMock(project=project, relative_dirs=relative_dirs, extension=extension) - result.__iter__.side_effect = lambda: iter(filesystem_iter(result)) - return result - self.mock_filesystem_constructor.side_effect = create_filesystem_searcher + self.mock_filesystem_search = self.filesystem_search.start() + self.mock_filesystem_search.side_effect = mock_filesystem_search # Create HookParser patcher self.hook_patcher = patch.object(