diff --git a/fawltydeps/check.py b/fawltydeps/check.py index a5b9a893a..f48139d95 100644 --- a/fawltydeps/check.py +++ b/fawltydeps/check.py @@ -1,93 +1,21 @@ -"Compare imports and dependencies" +"""Compare imports and dependencies to determine undeclared and unused deps.""" import logging -import sys from itertools import groupby -from typing import Dict, Iterable, List, Optional, Tuple +from typing import Dict, List +from fawltydeps.packages import Package from fawltydeps.settings import Settings from fawltydeps.types import ( DeclaredDependency, - DependenciesMapping, - Package, ParsedImport, UndeclaredDependency, UnusedDependency, ) -# importlib.metadata.packages_distributions() was introduced in v3.10, but it -# is not able to infer import names for modules lacking a top_level.txt until -# v3.11. Hence we prefer importlib_metadata in v3.10 as well as pre-v3.10. -if sys.version_info >= (3, 11): - from importlib.metadata import packages_distributions -else: - from importlib_metadata import packages_distributions - logger = logging.getLogger(__name__) -class LocalPackageLookup: - """Lookup import names exposed by packages installed in the current venv.""" - - def __init__(self) -> None: - """Collect packages installed in the current python environment. - - Use importlib.metadata to look up the mapping between packages and their - provided import names. This obviously depends on the Python environment - (e.g. virtualenv) that we're calling from. - """ - # We call packages_distributions() only _once here, and build a cache of - # Package objects from the information extracted. - self.packages: Dict[str, Package] = {} - for import_name, package_names in packages_distributions().items(): - for package_name in package_names: - package = self.packages.setdefault( - Package.normalize_name(package_name), - Package(package_name), - ) - package.add_import_names( - import_name, mapping=DependenciesMapping.LOCAL_ENV - ) - - def lookup_package(self, package_name: str) -> Optional[Package]: - """Convert a package name to a locally available Package object. - - (Although this function generally works with _all_ locally available - packages, we apply it only to the subset that is the dependencies of - the current project.) - - Return the Package object that encapsulates the package-name-to-import- - names mapping for the given package name. - - Return None if we're unable to find any import names for the given - package name. This is typically because the package is missing from the - current environment, or because we fail to determine its provided import - names. - """ - return self.packages.get(Package.normalize_name(package_name)) - - -def resolve_dependencies(dep_names: Iterable[str]) -> Dict[str, Package]: - """Associate dependencies with corresponding Package objects. - - Use LocalPackageLookup to find Package objects for each of the given - dependencies. For dependencies that cannot be found with LocalPackageLookup, - fabricate an identity mapping (a pseudo-package making available an import - of the same name as the package, modulo normalization). - - Return a dict mapping dependency names to the resolved Package objects. - """ - ret = {} - local_packages = LocalPackageLookup() - for name in dep_names: - if name not in ret: - package = local_packages.lookup_package(name) - if package is None: # fall back to identity mapping - package = Package.identity_mapping(name) - ret[name] = package - return ret - - def calculate_undeclared( imports: List[ParsedImport], resolved_deps: Dict[str, Package], diff --git a/fawltydeps/main.py b/fawltydeps/main.py index 7142dfc90..f1896067a 100644 --- a/fawltydeps/main.py +++ b/fawltydeps/main.py @@ -23,12 +23,9 @@ from pydantic.json import custom_pydantic_encoder # pylint: disable=no-name-in-module from fawltydeps import extract_imports -from fawltydeps.check import ( - calculate_undeclared, - calculate_unused, - resolve_dependencies, -) +from fawltydeps.check import calculate_undeclared, calculate_unused from fawltydeps.extract_declared_dependencies import extract_declared_dependencies +from fawltydeps.packages import Package, resolve_dependencies from fawltydeps.settings import ( Action, OutputFormat, @@ -38,7 +35,6 @@ ) from fawltydeps.types import ( DeclaredDependency, - Package, ParsedImport, UndeclaredDependency, UnparseablePathException, diff --git a/fawltydeps/packages.py b/fawltydeps/packages.py new file mode 100644 index 000000000..96a5be504 --- /dev/null +++ b/fawltydeps/packages.py @@ -0,0 +1,158 @@ +"""Encapsulate the lookup of packages and their provided import names.""" + +import logging +import sys +from dataclasses import dataclass, field +from enum import Enum +from typing import Dict, Iterable, Optional, Set + +from fawltydeps.utils import hide_dataclass_fields + +# importlib.metadata.packages_distributions() was introduced in v3.10, but it +# is not able to infer import names for modules lacking a top_level.txt until +# v3.11. Hence we prefer importlib_metadata in v3.10 as well as pre-v3.10. +if sys.version_info >= (3, 11): + from importlib.metadata import packages_distributions +else: + from importlib_metadata import packages_distributions + +logger = logging.getLogger(__name__) + + +class DependenciesMapping(str, Enum): + """Types of dependency and imports mapping""" + + IDENTITY = "identity" + LOCAL_ENV = "local_env" + + +@dataclass +class Package: + """Encapsulate an installable Python package. + + This encapsulates the mapping between a package name (i.e. something you can + pass to `pip install`) and the import names that it provides once it is + installed. + """ + + package_name: str + mappings: Dict[DependenciesMapping, Set[str]] = field(default_factory=dict) + import_names: Set[str] = field(default_factory=set) + + def __post_init__(self) -> None: + # The .import_names member is entirely redundant, as it can always be + # calculated from a union of self.mappings.values(). However, it is + # still used often enough (.is_used() is called once per declared + # dependency) that it makes sense to pre-calculate it, and rather hide + # the redundancy from our JSON output + self.import_names = {name for names in self.mappings.values() for name in names} + hide_dataclass_fields(self, "import_names") + + @staticmethod + def normalize_name(package_name: str) -> str: + """Perform standard normalization of package names. + + Verbatim package names are not always appropriate to use in various + contexts: For example, a package can be installed using one spelling + (e.g. typing-extensions), but once installed, it is presented in the + context of the local environment with a slightly different spelling + (e.g. typing_extension). + """ + return package_name.lower().replace("-", "_") + + def add_import_names( + self, *import_names: str, mapping: DependenciesMapping + ) -> None: + """Add import names provided by this package. + + Import names must be associated with a DependenciesMapping enum value, + as keeping track of this is extremely helpful when debugging. + """ + self.mappings.setdefault(mapping, set()).update(import_names) + self.import_names.update(import_names) + + def add_identity_import(self) -> None: + """Add identity mapping to this package. + + This builds on an assumption that a package 'foo' installed with e.g. + `pip install foo`, will also provide an import name 'foo'. This + assumption does not always hold, but sometimes we don't have much else + to go on... + """ + self.add_import_names( + self.normalize_name(self.package_name), + mapping=DependenciesMapping.IDENTITY, + ) + + @classmethod + def identity_mapping(cls, package_name: str) -> "Package": + """Factory for conveniently creating identity-mapped package object.""" + ret = cls(package_name) + ret.add_identity_import() + return ret + + def is_used(self, imported_names: Iterable[str]) -> bool: + """Return True iff this package is among the given import names.""" + return bool(self.import_names.intersection(imported_names)) + + +class LocalPackageLookup: + """Lookup import names exposed by packages installed in the current venv.""" + + def __init__(self) -> None: + """Collect packages installed in the current python environment. + + Use importlib.metadata to look up the mapping between packages and their + provided import names. This obviously depends on the Python environment + (e.g. virtualenv) that we're calling from. + """ + # We call packages_distributions() only _once here, and build a cache of + # Package objects from the information extracted. + self.packages: Dict[str, Package] = {} + for import_name, package_names in packages_distributions().items(): + for package_name in package_names: + package = self.packages.setdefault( + Package.normalize_name(package_name), + Package(package_name), + ) + package.add_import_names( + import_name, mapping=DependenciesMapping.LOCAL_ENV + ) + + def lookup_package(self, package_name: str) -> Optional[Package]: + """Convert a package name to a locally available Package object. + + (Although this function generally works with _all_ locally available + packages, we apply it only to the subset that is the dependencies of + the current project.) + + Return the Package object that encapsulates the package-name-to-import- + names mapping for the given package name. + + Return None if we're unable to find any import names for the given + package name. This is typically because the package is missing from the + current environment, or because we fail to determine its provided import + names. + """ + return self.packages.get(Package.normalize_name(package_name)) + + +def resolve_dependencies(dep_names: Iterable[str]) -> Dict[str, Package]: + """Associate dependencies with corresponding Package objects. + + Use LocalPackageLookup to find Package objects for each of the given + dependencies. For dependencies that cannot be found with LocalPackageLookup, + fabricate an identity mapping (a pseudo-package making available an import + of the same name as the package, modulo normalization). + + Return a dict mapping dependency names to the resolved Package objects. + """ + ret = {} + local_packages = LocalPackageLookup() + for name in dep_names: + if name not in ret: + package = local_packages.lookup_package(name) + if package is None: # fall back to identity mapping + package = Package.identity_mapping(name) + ret[name] = package + return ret diff --git a/fawltydeps/types.py b/fawltydeps/types.py index d3072c1f4..990f710c2 100644 --- a/fawltydeps/types.py +++ b/fawltydeps/types.py @@ -2,10 +2,9 @@ import sys from dataclasses import asdict, dataclass, field, replace -from enum import Enum from functools import total_ordering from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from fawltydeps.utils import hide_dataclass_fields @@ -124,83 +123,6 @@ class DeclaredDependency: source: Location -class DependenciesMapping(str, Enum): - """Types of dependency and imports mapping""" - - IDENTITY = "identity" - LOCAL_ENV = "local_env" - - -@dataclass -class Package: - """Encapsulate an installable Python package. - - This encapsulates the mapping between a package name (i.e. something you can - pass to `pip install`) and the import names that it provides once it is - installed. - """ - - package_name: str - mappings: Dict[DependenciesMapping, Set[str]] = field(default_factory=dict) - import_names: Set[str] = field(default_factory=set) - - def __post_init__(self) -> None: - # The .import_names member is entirely redundant, as it can always be - # calculated from a union of self.mappings.values(). However, it is - # still used often enough (.is_used() is called once per declared - # dependency) that it makes sense to pre-calculate it, and rather hide - # the redundancy from our JSON output - self.import_names = {name for names in self.mappings.values() for name in names} - hide_dataclass_fields(self, "import_names") - - @staticmethod - def normalize_name(package_name: str) -> str: - """Perform standard normalization of package names. - - Verbatim package names are not always appropriate to use in various - contexts: For example, a package can be installed using one spelling - (e.g. typing-extensions), but once installed, it is presented in the - context of the local environment with a slightly different spelling - (e.g. typing_extension). - """ - return package_name.lower().replace("-", "_") - - def add_import_names( - self, *import_names: str, mapping: DependenciesMapping - ) -> None: - """Add import names provided by this package. - - Import names must be associated with a DependenciesMapping enum value, - as keeping track of this is extremely helpful when debugging. - """ - self.mappings.setdefault(mapping, set()).update(import_names) - self.import_names.update(import_names) - - def add_identity_import(self) -> None: - """Add identity mapping to this package. - - This builds on an assumption that a package 'foo' installed with e.g. - `pip install foo`, will also provide an import name 'foo'. This - assumption does not always hold, but sometimes we don't have much else - to go on... - """ - self.add_import_names( - self.normalize_name(self.package_name), - mapping=DependenciesMapping.IDENTITY, - ) - - @classmethod - def identity_mapping(cls, package_name: str) -> "Package": - """Factory for conveniently creating identity-mapped package object.""" - ret = cls(package_name) - ret.add_identity_import() - return ret - - def is_used(self, imported_names: Iterable[str]) -> bool: - """Return True iff this package is among the given import names.""" - return bool(self.import_names.intersection(imported_names)) - - @dataclass class UndeclaredDependency: """Undeclared dependency found by analysis in the 'check' module.""" diff --git a/tests/test_compare_imports_to_dependencies.py b/tests/test_compare_imports_to_dependencies.py index e9b0768cd..c7e0e4309 100644 --- a/tests/test_compare_imports_to_dependencies.py +++ b/tests/test_compare_imports_to_dependencies.py @@ -1,259 +1,20 @@ """Test the imports to dependencies comparison function.""" -from dataclasses import dataclass, field -from pathlib import Path -from typing import Dict, List - import pytest -from fawltydeps.check import ( - LocalPackageLookup, - calculate_undeclared, - calculate_unused, - resolve_dependencies, -) +from fawltydeps.check import calculate_undeclared, calculate_unused from fawltydeps.settings import Settings -from fawltydeps.types import ( - DeclaredDependency, - DependenciesMapping, - Location, - Package, - ParsedImport, - UndeclaredDependency, - UnusedDependency, -) - -from .utils import deps_factory - -# TODO: These tests are not fully isolated, i.e. they do not control the -# virtualenv in which they run. For now, we assume that we are running in an -# environment where at least these packages are available: -# - setuptools (exposes multiple import names, including pkg_resources) -# - pip (exposes a single import name: pip) -# - isort (exposes no top_level.txt, but 'isort' import name can be inferred) - -local_env = LocalPackageLookup() - - -def imports_factory(*imports: str) -> List[ParsedImport]: - return [ParsedImport(imp, Location("")) for imp in imports] - - -def resolved_factory(*deps: str) -> Dict[str, Package]: - def mapping_for_dep(dep: str) -> DependenciesMapping: - if local_env.lookup_package(dep) is None: - return DependenciesMapping.IDENTITY - return DependenciesMapping.LOCAL_ENV - - return {dep: Package(dep, {mapping_for_dep(dep): {dep}}) for dep in deps} - - -def undeclared_factory(*deps: str) -> List[UndeclaredDependency]: - return [UndeclaredDependency(dep, [Location("")]) for dep in deps] - - -def unused_factory(*deps: str) -> List[UnusedDependency]: - return [UnusedDependency(dep, [Location(Path("foo"))]) for dep in deps] - - -@dataclass -class FDTestVector: # pylint: disable=too-many-instance-attributes - """Test vectors for various parts of the FawltyDeps core logic.""" - - id: str - imports: List[ParsedImport] = field(default_factory=list) - declared_deps: List[DeclaredDependency] = field(default_factory=list) - ignore_unused: List[str] = field(default_factory=list) - ignore_undeclared: List[str] = field(default_factory=list) - expect_resolved_deps: Dict[str, Package] = field(default_factory=dict) - expect_undeclared_deps: List[UndeclaredDependency] = field(default_factory=list) - expect_unused_deps: List[UnusedDependency] = field(default_factory=list) - - -testdata = [ - FDTestVector("no_imports_no_deps"), - FDTestVector( - "one_import_no_deps", - imports=imports_factory("pandas"), - expect_undeclared_deps=undeclared_factory("pandas"), - ), - FDTestVector( - "no_imports_one_dep", - declared_deps=deps_factory("pandas"), - expect_resolved_deps=resolved_factory("pandas"), - expect_unused_deps=unused_factory("pandas"), - ), - FDTestVector( - "matched_import_with_dep", - imports=imports_factory("pandas"), - declared_deps=deps_factory("pandas"), - expect_resolved_deps=resolved_factory("pandas"), - ), - FDTestVector( - "mixed_imports_with_unused_and_undeclared_deps", - imports=imports_factory("pandas", "numpy"), - declared_deps=deps_factory("pandas", "scipy"), - expect_resolved_deps=resolved_factory("pandas", "scipy"), - expect_undeclared_deps=undeclared_factory("numpy"), - expect_unused_deps=unused_factory("scipy"), - ), - FDTestVector( - "mixed_imports_from_diff_files_with_unused_and_undeclared_deps", - imports=imports_factory("pandas") - + [ParsedImport("numpy", Location(Path("my_file.py"), lineno=3))], - declared_deps=deps_factory("pandas", "scipy"), - expect_resolved_deps=resolved_factory("pandas", "scipy"), - expect_undeclared_deps=[ - UndeclaredDependency( - "numpy", - [Location(Path("my_file.py"), lineno=3)], - ) - ], - expect_unused_deps=unused_factory("scipy"), - ), - FDTestVector( - "unused_dep_that_is_ignore_unused__not_reported_as_unused", - declared_deps=deps_factory("pip"), - ignore_unused=["pip"], - expect_resolved_deps=resolved_factory("pip"), - ), - FDTestVector( - "used_dep_that_is_ignore_unused__not_reported_as_unused", - imports=imports_factory("isort"), - declared_deps=deps_factory("isort"), - ignore_unused=["isort"], - expect_resolved_deps=resolved_factory("isort"), - ), - FDTestVector( - "undeclared_dep_that_is_ignore_unused__reported_as_undeclared", - imports=imports_factory("isort"), - ignore_unused=["isort"], - expect_undeclared_deps=undeclared_factory("isort"), - ), - FDTestVector( - "mixed_deps__report_undeclared_and_non_ignored_unused", - imports=imports_factory("pandas", "numpy"), - declared_deps=deps_factory("pandas", "isort", "flake8"), - ignore_unused=["isort"], - expect_resolved_deps=resolved_factory("pandas", "isort", "flake8"), - expect_undeclared_deps=undeclared_factory("numpy"), - expect_unused_deps=unused_factory("flake8"), - ), - FDTestVector( - "undeclared_dep_that_is_ignore_undeclared__not_reported_as_undeclared", - imports=imports_factory("invalid_import"), - ignore_undeclared=["invalid_import"], - ), - FDTestVector( - "declared_dep_that_is_ignore_undeclared__not_reported_as_undeclared", - imports=imports_factory("isort"), - declared_deps=deps_factory("isort"), - ignore_undeclared=["isort"], - expect_resolved_deps=resolved_factory("isort"), - ), - FDTestVector( - "unused_dep_that_is_ignore_undeclared__reported_as_unused", - declared_deps=deps_factory("isort"), - ignore_undeclared=["isort"], - expect_resolved_deps=resolved_factory("isort"), - expect_unused_deps=unused_factory("isort"), - ), - FDTestVector( - "mixed_deps__report_unused_and_non_ignored_undeclared", - imports=imports_factory("pandas", "numpy", "not_valid"), - declared_deps=deps_factory("pandas", "flake8"), - ignore_undeclared=["not_valid"], - expect_resolved_deps=resolved_factory("pandas", "flake8"), - expect_undeclared_deps=undeclared_factory("numpy"), - expect_unused_deps=unused_factory("flake8"), - ), - FDTestVector( - "mixed_deps__report_only_non_ignored_unused_and_non_ignored_undeclared", - imports=imports_factory("pandas", "numpy", "not_valid"), - declared_deps=deps_factory("pandas", "flake8", "isort"), - ignore_undeclared=["not_valid"], - ignore_unused=["isort"], - expect_resolved_deps=resolved_factory("pandas", "flake8", "isort"), - expect_undeclared_deps=undeclared_factory("numpy"), - expect_unused_deps=unused_factory("flake8"), - ), - FDTestVector( - "deps_with_diff_name_for_the_same_import", - declared_deps=[ - DeclaredDependency(name="Pip", source=Location(Path("requirements1.txt"))), - DeclaredDependency(name="pip", source=Location(Path("requirements2.txt"))), - ], - expect_resolved_deps={ - "Pip": Package("pip", {DependenciesMapping.LOCAL_ENV: {"pip"}}), - "pip": Package("pip", {DependenciesMapping.LOCAL_ENV: {"pip"}}), - }, - expect_unused_deps=[ - UnusedDependency("Pip", [Location(Path("requirements1.txt"))]), - UnusedDependency("pip", [Location(Path("requirements2.txt"))]), - ], - ), -] - - -@pytest.mark.parametrize( - "dep_names,expected", - [ - pytest.param([], {}, id="no_deps__empty_dict"), - pytest.param( - ["pandas", "numpy", "other"], - { - "pandas": Package("pandas", {DependenciesMapping.IDENTITY: {"pandas"}}), - "numpy": Package("numpy", {DependenciesMapping.IDENTITY: {"numpy"}}), - "other": Package("other", {DependenciesMapping.IDENTITY: {"other"}}), - }, - id="uninstalled_deps__use_identity_mapping", - ), - pytest.param( - ["pandas", "numpy", "other"], - { - "pandas": Package("pandas", {DependenciesMapping.IDENTITY: {"pandas"}}), - "numpy": Package("numpy", {DependenciesMapping.IDENTITY: {"numpy"}}), - "other": Package("other", {DependenciesMapping.IDENTITY: {"other"}}), - }, - id="uninstalled_deps__use_identity_mapping", - ), - pytest.param( - ["setuptools", "pip", "isort"], - { - "setuptools": Package( - "setuptools", - { - DependenciesMapping.LOCAL_ENV: { - "_distutils_hack", - "pkg_resources", - "setuptools", - } - }, - ), - "pip": Package("pip", {DependenciesMapping.LOCAL_ENV: {"pip"}}), - "isort": Package("isort", {DependenciesMapping.LOCAL_ENV: {"isort"}}), - }, - id="installed_deps__use_local_env_mapping", - ), - ], -) -def test_resolve_dependencies__focus_on_mappings(dep_names, expected): - assert resolve_dependencies(dep_names) == expected - -@pytest.mark.parametrize("vector", [pytest.param(v, id=v.id) for v in testdata]) -def test_resolve_dependencies(vector): - dep_names = [dd.name for dd in vector.declared_deps] - assert resolve_dependencies(dep_names) == vector.expect_resolved_deps +from .utils import test_vectors -@pytest.mark.parametrize("vector", [pytest.param(v, id=v.id) for v in testdata]) +@pytest.mark.parametrize("vector", [pytest.param(v, id=v.id) for v in test_vectors]) def test_calculate_undeclared(vector): settings = Settings(ignore_undeclared=vector.ignore_undeclared) actual = calculate_undeclared(vector.imports, vector.expect_resolved_deps, settings) assert actual == vector.expect_undeclared_deps -@pytest.mark.parametrize("vector", [pytest.param(v, id=v.id) for v in testdata]) +@pytest.mark.parametrize("vector", [pytest.param(v, id=v.id) for v in test_vectors]) def test_calculate_unused(vector): settings = Settings(ignore_unused=vector.ignore_unused) actual = calculate_unused( diff --git a/tests/test_map_dep_name_to_import_names.py b/tests/test_map_dep_name_to_import_names.py deleted file mode 100644 index 0b2e7b4b8..000000000 --- a/tests/test_map_dep_name_to_import_names.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Test the mapping of dependency names to import names.""" - - -import pytest - -from fawltydeps.check import LocalPackageLookup, resolve_dependencies -from fawltydeps.types import DependenciesMapping, Package - -# TODO: These tests are not fully isolated, i.e. they do not control the -# virtualenv in which they run. For now, we assume that we are running in an -# environment where at least these packages are available: -# - setuptools (exposes multiple import names, including pkg_resources) -# - pip (exposes a single import name: pip) -# - isort (exposes no top_level.txt, but 'isort' import name can be inferred) - - -@pytest.mark.parametrize( - "dep_name,expect_import_names", - [ - pytest.param( - "NOT_A_PACKAGE", - None, - id="missing_package__returns_None", - ), - pytest.param( - "isort", - {"isort"}, - id="package_exposes_nothing__can_still_infer_import_name", - ), - pytest.param( - "pip", - {"pip"}, - id="package_exposes_one_entry__returns_entry", - ), - pytest.param( - "setuptools", - {"_distutils_hack", "pkg_resources", "setuptools"}, - id="package_exposes_many_entries__returns_all_entries", - ), - pytest.param( - "SETUPTOOLS", - {"_distutils_hack", "pkg_resources", "setuptools"}, - id="package_declared_in_capital_letters__is_successfully_mapped_with_d2i", - ), - pytest.param( - "typing-extensions", - {"typing_extensions"}, - id="package_with_hyphen__provides_import_name_with_underscore", - ), - ], -) -def test_LocalPackageLookup_lookup_package(dep_name, expect_import_names): - lpl = LocalPackageLookup() - actual = lpl.lookup_package(dep_name) - if expect_import_names is None: - assert actual is None - else: - assert actual.import_names == expect_import_names - - -@pytest.mark.parametrize( - "dep_names,expected_packages", - [ - pytest.param( - ["pip"], - {"pip": Package("pip", {DependenciesMapping.LOCAL_ENV: {"pip"}})}, - id="dependency_present_in_local_env__uses_d2i_mapping", - ), - pytest.param( - ["pandas"], - {"pandas": Package("pandas", {DependenciesMapping.IDENTITY: {"pandas"}})}, - id="dependency_not_present_in_local_env__uses_id_mapping", - ), - pytest.param( - ["pandas", "pip"], - { - "pip": Package("pip", {DependenciesMapping.LOCAL_ENV: {"pip"}}), - "pandas": Package("pandas", {DependenciesMapping.IDENTITY: {"pandas"}}), - }, - id="mixed_dependencies_in_local_env__uses_id_and_d2i_mapping", - ), - pytest.param( - ["setuptools"], - { - "setuptools": Package( - "setuptools", - { - DependenciesMapping.LOCAL_ENV: { - "_distutils_hack", - "pkg_resources", - "setuptools", - } - }, - ), - }, - id="dependency_present_in_local_env__uses_d2i_mapping_and_has_correct_imports", - ), - ], -) -def test_resolve_dependencies(dep_names, expected_packages): - assert resolve_dependencies(dep_names) == expected_packages diff --git a/tests/test_packages.py b/tests/test_packages.py new file mode 100644 index 000000000..7e9d32ce4 --- /dev/null +++ b/tests/test_packages.py @@ -0,0 +1,241 @@ +"""Verify behavior of package lookup and mapping to import names.""" + +import pytest + +from fawltydeps.packages import ( + DependenciesMapping, + LocalPackageLookup, + Package, + resolve_dependencies, +) + +from .utils import test_vectors + + +def test_package__empty_package__matches_nothing(): + p = Package("foobar") # no mappings + assert p.package_name == "foobar" + assert not p.is_used(["foobar"]) + + +@pytest.mark.parametrize( + "package_name,matching_imports,non_matching_imports", + [ + pytest.param( + "foobar", + ["foobar", "and", "other", "names"], + ["only", "other", "names", "foo_bar", "Foobar", "FooBar", "FOOBAR"], + id="simple_lowercase_name__matches_itself_only", + ), + pytest.param( + "FooBar", + ["foobar", "and", "other", "names"], + ["only", "other", "names", "foo_bar", "Foobar", "FooBar", "FOOBAR"], + id="mixed_case_name__matches_lowercase_only", + ), + pytest.param( + "typing-extensions", + ["typing_extensions", "and", "other", "names"], + ["typing-extensions", "typingextensions"], + id="name_with_hyphen__matches_name_with_underscore_only", + ), + pytest.param( + "Foo-Bar", + ["foo_bar", "and", "other", "names"], + ["foo-bar", "Foobar", "FooBar", "FOOBAR"], + id="weird_name__matches_normalized_name_only", + ), + ], +) +def test_package__identity_mapping( + package_name, matching_imports, non_matching_imports +): + p = Package.identity_mapping(package_name) + assert p.package_name == package_name # package name is not normalized + assert p.is_used(matching_imports) + assert not p.is_used(non_matching_imports) + + +@pytest.mark.parametrize( + "package_name,import_names,matching_imports,non_matching_imports", + [ + pytest.param( + "foobar", + ["foobar"], + ["foobar", "and", "other", "names"], + ["only", "other", "names", "foo_bar", "Foobar", "FooBar", "FOOBAR"], + id="simple_name_mapped_to_itself__matches_itself_only", + ), + pytest.param( + "FooBar", + ["FooBar"], + ["FooBar", "and", "other", "names"], + ["only", "other", "names", "foo_bar", "foobar", "FOOBAR"], + id="mixed_case_name_mapped_to_itself__matches_exact_spelling_only", + ), + pytest.param( + "typing-extensions", + ["typing_extensions"], + ["typing_extensions", "and", "other", "names"], + ["typing-extensions", "typingextensions"], + id="hyphen_name_mapped_to_underscore_name__matches_only_underscore_name", + ), + pytest.param( + "Foo-Bar", + ["blorp"], + ["blorp", "and", "other", "names"], + ["Foo-Bar", "foo-bar", "foobar", "FooBar", "FOOBAR", "Blorp", "BLORP"], + id="weird_name_mapped_diff_name__matches_diff_name_only", + ), + pytest.param( + "foobar", + ["foo", "bar", "baz"], + ["foo", "and", "other", "names"], + ["foobar", "and", "other", "names"], + id="name_with_three_imports__matches_first_import", + ), + pytest.param( + "foobar", + ["foo", "bar", "baz"], + ["bar", "and", "other", "names"], + ["foobar", "and", "other", "names"], + id="name_with_three_imports__matches_second_import", + ), + pytest.param( + "foobar", + ["foo", "bar", "baz"], + ["baz", "and", "other", "names"], + ["foobar", "and", "other", "names"], + id="name_with_three_imports__matches_third_import", + ), + ], +) +def test_package__local_env_mapping( + package_name, import_names, matching_imports, non_matching_imports +): + p = Package(package_name) + p.add_import_names(*import_names, mapping=DependenciesMapping.LOCAL_ENV) + assert p.package_name == package_name # package name is not normalized + assert p.is_used(matching_imports) + assert not p.is_used(non_matching_imports) + + +def test_package__both_mappings(): + p = Package.identity_mapping("FooBar") + import_names = ["foo", "bar", "baz"] + p.add_import_names(*import_names, mapping=DependenciesMapping.LOCAL_ENV) + assert p.package_name == "FooBar" # package name is not normalized + assert p.is_used(["foobar"]) # but identity-mapped import name _is_. + assert p.is_used(["foo"]) + assert p.is_used(["bar"]) + assert p.is_used(["baz"]) + assert not p.is_used(["fooba"]) + assert not p.is_used(["foobarbaz"]) + assert p.mappings == { + DependenciesMapping.IDENTITY: {"foobar"}, + DependenciesMapping.LOCAL_ENV: {"foo", "bar", "baz"}, + } + assert p.import_names == {"foobar", "foo", "bar", "baz"} + + +# TODO: These tests are not fully isolated, i.e. they do not control the +# virtualenv in which they run. For now, we assume that we are running in an +# environment where at least these packages are available: +# - setuptools (exposes multiple import names, including pkg_resources) +# - pip (exposes a single import name: pip) +# - isort (exposes no top_level.txt, but 'isort' import name can be inferred) + + +@pytest.mark.parametrize( + "dep_name,expect_import_names", + [ + pytest.param( + "NOT_A_PACKAGE", + None, + id="missing_package__returns_None", + ), + pytest.param( + "isort", + {"isort"}, + id="package_exposes_nothing__can_still_infer_import_name", + ), + pytest.param( + "pip", + {"pip"}, + id="package_exposes_one_entry__returns_entry", + ), + pytest.param( + "setuptools", + {"_distutils_hack", "pkg_resources", "setuptools"}, + id="package_exposes_many_entries__returns_all_entries", + ), + pytest.param( + "SETUPTOOLS", + {"_distutils_hack", "pkg_resources", "setuptools"}, + id="package_declared_in_capital_letters__is_successfully_mapped_with_d2i", + ), + pytest.param( + "typing-extensions", + {"typing_extensions"}, + id="package_with_hyphen__provides_import_name_with_underscore", + ), + ], +) +def test_LocalPackageLookup_lookup_package(dep_name, expect_import_names): + lpl = LocalPackageLookup() + actual = lpl.lookup_package(dep_name) + if expect_import_names is None: + assert actual is None + else: + assert actual.import_names == expect_import_names + + +@pytest.mark.parametrize( + "dep_names,expected", + [ + pytest.param([], {}, id="no_deps__empty_dict"), + pytest.param( + ["pandas", "numpy", "other"], + { + "pandas": Package("pandas", {DependenciesMapping.IDENTITY: {"pandas"}}), + "numpy": Package("numpy", {DependenciesMapping.IDENTITY: {"numpy"}}), + "other": Package("other", {DependenciesMapping.IDENTITY: {"other"}}), + }, + id="uninstalled_deps__use_identity_mapping", + ), + pytest.param( + ["setuptools", "pip", "isort"], + { + "setuptools": Package( + "setuptools", + { + DependenciesMapping.LOCAL_ENV: { + "_distutils_hack", + "pkg_resources", + "setuptools", + } + }, + ), + "pip": Package("pip", {DependenciesMapping.LOCAL_ENV: {"pip"}}), + "isort": Package("isort", {DependenciesMapping.LOCAL_ENV: {"isort"}}), + }, + id="installed_deps__use_local_env_mapping", + ), + pytest.param( + ["pandas", "pip"], + { + "pip": Package("pip", {DependenciesMapping.LOCAL_ENV: {"pip"}}), + "pandas": Package("pandas", {DependenciesMapping.IDENTITY: {"pandas"}}), + }, + id="mixed_deps__uses_mixture_of_identity_and_local_env_mapping", + ), + ], +) +def test_resolve_dependencies__focus_on_mappings(dep_names, expected): + assert resolve_dependencies(dep_names) == expected + + +@pytest.mark.parametrize("vector", [pytest.param(v, id=v.id) for v in test_vectors]) +def test_resolve_dependencies(vector): + dep_names = [dd.name for dd in vector.declared_deps] + assert resolve_dependencies(dep_names) == vector.expect_resolved_deps diff --git a/tests/test_types.py b/tests/test_types.py index 3bd8437c9..2a4bcc300 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -5,13 +5,7 @@ import pytest -from fawltydeps.types import ( - DeclaredDependency, - DependenciesMapping, - Location, - Package, - ParsedImport, -) +from fawltydeps.types import DeclaredDependency, Location, ParsedImport testdata = { # Test ID -> (Location args, expected string representation, sort order) # First arg must be a Path, or "" @@ -115,129 +109,3 @@ def test_declareddependency_is_immutable(): dd.name = "bar_package" with pytest.raises(FrozenInstanceError): dd.source = dd.source.supply(lineno=123) - - -def test_package__empty_package__matches_nothing(): - p = Package("foobar") # no mappings - assert p.package_name == "foobar" - assert not p.is_used(["foobar"]) - - -@pytest.mark.parametrize( - "package_name,matching_imports,non_matching_imports", - [ - pytest.param( - "foobar", - ["foobar", "and", "other", "names"], - ["only", "other", "names", "foo_bar", "Foobar", "FooBar", "FOOBAR"], - id="simple_lowercase_name__matches_itself_only", - ), - pytest.param( - "FooBar", - ["foobar", "and", "other", "names"], - ["only", "other", "names", "foo_bar", "Foobar", "FooBar", "FOOBAR"], - id="mixed_case_name__matches_lowercase_only", - ), - pytest.param( - "typing-extensions", - ["typing_extensions", "and", "other", "names"], - ["typing-extensions", "typingextensions"], - id="name_with_hyphen__matches_name_with_underscore_only", - ), - pytest.param( - "Foo-Bar", - ["foo_bar", "and", "other", "names"], - ["foo-bar", "Foobar", "FooBar", "FOOBAR"], - id="weird_name__matches_normalized_name_only", - ), - ], -) -def test_package__identity_mapping( - package_name, matching_imports, non_matching_imports -): - p = Package.identity_mapping(package_name) - assert p.package_name == package_name # package name is not normalized - assert p.is_used(matching_imports) - assert not p.is_used(non_matching_imports) - - -@pytest.mark.parametrize( - "package_name,import_names,matching_imports,non_matching_imports", - [ - pytest.param( - "foobar", - ["foobar"], - ["foobar", "and", "other", "names"], - ["only", "other", "names", "foo_bar", "Foobar", "FooBar", "FOOBAR"], - id="simple_name_mapped_to_itself__matches_itself_only", - ), - pytest.param( - "FooBar", - ["FooBar"], - ["FooBar", "and", "other", "names"], - ["only", "other", "names", "foo_bar", "foobar", "FOOBAR"], - id="mixed_case_name_mapped_to_itself__matches_exact_spelling_only", - ), - pytest.param( - "typing-extensions", - ["typing_extensions"], - ["typing_extensions", "and", "other", "names"], - ["typing-extensions", "typingextensions"], - id="hyphen_name_mapped_to_underscore_name__matches_only_underscore_name", - ), - pytest.param( - "Foo-Bar", - ["blorp"], - ["blorp", "and", "other", "names"], - ["Foo-Bar", "foo-bar", "foobar", "FooBar", "FOOBAR", "Blorp", "BLORP"], - id="weird_name_mapped_diff_name__matches_diff_name_only", - ), - pytest.param( - "foobar", - ["foo", "bar", "baz"], - ["foo", "and", "other", "names"], - ["foobar", "and", "other", "names"], - id="name_with_three_imports__matches_first_import", - ), - pytest.param( - "foobar", - ["foo", "bar", "baz"], - ["bar", "and", "other", "names"], - ["foobar", "and", "other", "names"], - id="name_with_three_imports__matches_second_import", - ), - pytest.param( - "foobar", - ["foo", "bar", "baz"], - ["baz", "and", "other", "names"], - ["foobar", "and", "other", "names"], - id="name_with_three_imports__matches_third_import", - ), - ], -) -def test_package__local_env_mapping( - package_name, import_names, matching_imports, non_matching_imports -): - p = Package(package_name) - p.add_import_names(*import_names, mapping=DependenciesMapping.LOCAL_ENV) - assert p.package_name == package_name # package name is not normalized - assert p.is_used(matching_imports) - assert not p.is_used(non_matching_imports) - - -def test_package__both_mappings(): - p = Package.identity_mapping("FooBar") - import_names = ["foo", "bar", "baz"] - p.add_import_names(*import_names, mapping=DependenciesMapping.LOCAL_ENV) - assert p.package_name == "FooBar" # package name is not normalized - assert p.is_used(["foobar"]) # but identity-mapped import name _is_. - assert p.is_used(["foo"]) - assert p.is_used(["bar"]) - assert p.is_used(["baz"]) - assert not p.is_used(["fooba"]) - assert not p.is_used(["foobarbaz"]) - assert p.mappings == { - DependenciesMapping.IDENTITY: {"foobar"}, - DependenciesMapping.LOCAL_ENV: {"foo", "bar", "baz"}, - } - assert p.import_names == {"foobar", "foo", "bar", "baz"} diff --git a/tests/utils.py b/tests/utils.py index 14ed6e6ab..4e9773018 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,9 +1,17 @@ """ Utilities to share among test modules """ +from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Iterable, List +from typing import Any, Dict, Iterable, List -from fawltydeps.types import DeclaredDependency, Location +from fawltydeps.packages import DependenciesMapping, LocalPackageLookup, Package +from fawltydeps.types import ( + DeclaredDependency, + Location, + ParsedImport, + UndeclaredDependency, + UnusedDependency, +) def assert_unordered_equivalence(actual: Iterable[Any], expected: Iterable[Any]): @@ -14,5 +22,175 @@ def collect_dep_names(deps: Iterable[DeclaredDependency]) -> Iterable[str]: return (dep.name for dep in deps) +# TODO: These tests are not fully isolated, i.e. they do not control the +# virtualenv in which they run. For now, we assume that we are running in an +# environment where at least these packages are available: +# - setuptools (exposes multiple import names, including pkg_resources) +# - pip (exposes a single import name: pip) +# - isort (exposes no top_level.txt, but 'isort' import name can be inferred) + +local_env = LocalPackageLookup() + + +def imports_factory(*imports: str) -> List[ParsedImport]: + return [ParsedImport(imp, Location("")) for imp in imports] + + def deps_factory(*deps: str) -> List[DeclaredDependency]: return [DeclaredDependency(name=dep, source=Location(Path("foo"))) for dep in deps] + + +def resolved_factory(*deps: str) -> Dict[str, Package]: + def mapping_for_dep(dep: str) -> DependenciesMapping: + if local_env.lookup_package(dep) is None: + return DependenciesMapping.IDENTITY + return DependenciesMapping.LOCAL_ENV + + return {dep: Package(dep, {mapping_for_dep(dep): {dep}}) for dep in deps} + + +def undeclared_factory(*deps: str) -> List[UndeclaredDependency]: + return [UndeclaredDependency(dep, [Location("")]) for dep in deps] + + +def unused_factory(*deps: str) -> List[UnusedDependency]: + return [UnusedDependency(dep, [Location(Path("foo"))]) for dep in deps] + + +@dataclass +class FDTestVector: # pylint: disable=too-many-instance-attributes + """Test vectors for various parts of the FawltyDeps core logic.""" + + id: str + imports: List[ParsedImport] = field(default_factory=list) + declared_deps: List[DeclaredDependency] = field(default_factory=list) + ignore_unused: List[str] = field(default_factory=list) + ignore_undeclared: List[str] = field(default_factory=list) + expect_resolved_deps: Dict[str, Package] = field(default_factory=dict) + expect_undeclared_deps: List[UndeclaredDependency] = field(default_factory=list) + expect_unused_deps: List[UnusedDependency] = field(default_factory=list) + + +test_vectors = [ + FDTestVector("no_imports_no_deps"), + FDTestVector( + "one_import_no_deps", + imports=imports_factory("pandas"), + expect_undeclared_deps=undeclared_factory("pandas"), + ), + FDTestVector( + "no_imports_one_dep", + declared_deps=deps_factory("pandas"), + expect_resolved_deps=resolved_factory("pandas"), + expect_unused_deps=unused_factory("pandas"), + ), + FDTestVector( + "matched_import_with_dep", + imports=imports_factory("pandas"), + declared_deps=deps_factory("pandas"), + expect_resolved_deps=resolved_factory("pandas"), + ), + FDTestVector( + "mixed_imports_with_unused_and_undeclared_deps", + imports=imports_factory("pandas", "numpy"), + declared_deps=deps_factory("pandas", "scipy"), + expect_resolved_deps=resolved_factory("pandas", "scipy"), + expect_undeclared_deps=undeclared_factory("numpy"), + expect_unused_deps=unused_factory("scipy"), + ), + FDTestVector( + "mixed_imports_from_diff_files_with_unused_and_undeclared_deps", + imports=imports_factory("pandas") + + [ParsedImport("numpy", Location(Path("my_file.py"), lineno=3))], + declared_deps=deps_factory("pandas", "scipy"), + expect_resolved_deps=resolved_factory("pandas", "scipy"), + expect_undeclared_deps=[ + UndeclaredDependency( + "numpy", + [Location(Path("my_file.py"), lineno=3)], + ) + ], + expect_unused_deps=unused_factory("scipy"), + ), + FDTestVector( + "unused_dep_that_is_ignore_unused__not_reported_as_unused", + declared_deps=deps_factory("pip"), + ignore_unused=["pip"], + expect_resolved_deps=resolved_factory("pip"), + ), + FDTestVector( + "used_dep_that_is_ignore_unused__not_reported_as_unused", + imports=imports_factory("isort"), + declared_deps=deps_factory("isort"), + ignore_unused=["isort"], + expect_resolved_deps=resolved_factory("isort"), + ), + FDTestVector( + "undeclared_dep_that_is_ignore_unused__reported_as_undeclared", + imports=imports_factory("isort"), + ignore_unused=["isort"], + expect_undeclared_deps=undeclared_factory("isort"), + ), + FDTestVector( + "mixed_deps__report_undeclared_and_non_ignored_unused", + imports=imports_factory("pandas", "numpy"), + declared_deps=deps_factory("pandas", "isort", "flake8"), + ignore_unused=["isort"], + expect_resolved_deps=resolved_factory("pandas", "isort", "flake8"), + expect_undeclared_deps=undeclared_factory("numpy"), + expect_unused_deps=unused_factory("flake8"), + ), + FDTestVector( + "undeclared_dep_that_is_ignore_undeclared__not_reported_as_undeclared", + imports=imports_factory("invalid_import"), + ignore_undeclared=["invalid_import"], + ), + FDTestVector( + "declared_dep_that_is_ignore_undeclared__not_reported_as_undeclared", + imports=imports_factory("isort"), + declared_deps=deps_factory("isort"), + ignore_undeclared=["isort"], + expect_resolved_deps=resolved_factory("isort"), + ), + FDTestVector( + "unused_dep_that_is_ignore_undeclared__reported_as_unused", + declared_deps=deps_factory("isort"), + ignore_undeclared=["isort"], + expect_resolved_deps=resolved_factory("isort"), + expect_unused_deps=unused_factory("isort"), + ), + FDTestVector( + "mixed_deps__report_unused_and_non_ignored_undeclared", + imports=imports_factory("pandas", "numpy", "not_valid"), + declared_deps=deps_factory("pandas", "flake8"), + ignore_undeclared=["not_valid"], + expect_resolved_deps=resolved_factory("pandas", "flake8"), + expect_undeclared_deps=undeclared_factory("numpy"), + expect_unused_deps=unused_factory("flake8"), + ), + FDTestVector( + "mixed_deps__report_only_non_ignored_unused_and_non_ignored_undeclared", + imports=imports_factory("pandas", "numpy", "not_valid"), + declared_deps=deps_factory("pandas", "flake8", "isort"), + ignore_undeclared=["not_valid"], + ignore_unused=["isort"], + expect_resolved_deps=resolved_factory("pandas", "flake8", "isort"), + expect_undeclared_deps=undeclared_factory("numpy"), + expect_unused_deps=unused_factory("flake8"), + ), + FDTestVector( + "deps_with_diff_name_for_the_same_import", + declared_deps=[ + DeclaredDependency(name="Pip", source=Location(Path("requirements1.txt"))), + DeclaredDependency(name="pip", source=Location(Path("requirements2.txt"))), + ], + expect_resolved_deps={ + "Pip": Package("pip", {DependenciesMapping.LOCAL_ENV: {"pip"}}), + "pip": Package("pip", {DependenciesMapping.LOCAL_ENV: {"pip"}}), + }, + expect_unused_deps=[ + UnusedDependency("Pip", [Location(Path("requirements1.txt"))]), + UnusedDependency("pip", [Location(Path("requirements2.txt"))]), + ], + ), +]