From 4a5a7b991a5e9adae78c52bea61fd6e135728622 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Wed, 12 Feb 2025 17:35:41 +0100 Subject: [PATCH] Fix test fetcher (#36129) * fix * fix * update --------- Co-authored-by: ydshieh --- src/transformers/utils/import_utils.py | 5 +++++ utils/tests_fetcher.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/transformers/utils/import_utils.py b/src/transformers/utils/import_utils.py index 4aca04315659..c14ff2124aa0 100755 --- a/src/transformers/utils/import_utils.py +++ b/src/transformers/utils/import_utils.py @@ -1927,11 +1927,16 @@ def fetch__all__(file_content): if "__all__" not in file_content: return [] + start_index = None lines = file_content.splitlines() for index, line in enumerate(lines): if line.startswith("__all__"): start_index = index + # There is no line starting with `__all__` + if start_index is None: + return [] + lines = lines[start_index:] if not lines[0].startswith("__all__"): diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index c641ccb21e29..865b4fb9d5db 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -735,11 +735,23 @@ def get_module_dependencies(module_fname: str, cache: Dict[str, List[str]] = Non if module.endswith("__init__.py"): # So we get the imports from that init then try to find where our objects come from. new_imported_modules = extract_imports(module, cache=cache) + + # Add imports via `define_import_structure` after the #35167 as we remove explicit import in `__init__.py` + from transformers.utils.import_utils import define_import_structure + + new_imported_modules_2 = define_import_structure(PATH_TO_REPO / module) + + for mapping in new_imported_modules_2.values(): + for _module, _imports in mapping.items(): + _module = module.replace("__init__.py", f"{_module}.py") + new_imported_modules.append((_module, list(_imports))) + for new_module, new_imports in new_imported_modules: if any(i in new_imports for i in imports): if new_module not in dependencies: new_modules.append((new_module, [i for i in new_imports if i in imports])) imports = [i for i in imports if i not in new_imports] + if len(imports) > 0: # If there are any objects lefts, they may be a submodule path_to_module = PATH_TO_REPO / module.replace("__init__.py", "") @@ -759,6 +771,7 @@ def get_module_dependencies(module_fname: str, cache: Dict[str, List[str]] = Non dependencies.append(module) imported_modules = new_modules + return dependencies @@ -880,6 +893,7 @@ def create_reverse_dependency_map() -> Dict[str, List[str]]: depending on it recursively. This way the tests impacted by a change in file A are the test files in the list corresponding to key A in this result. """ + cache = {} # Start from the example deps init. example_deps, examples = init_test_examples_dependencies()