package.UserDefinedMapping: Include source of mapping in Package objects

When accumulating a user-defined Package object from multiple files
(e.g. a [tool.fawltydeps.custom_mapping] section in pyproject.toml
together with a --custom-mapping-file=mapping.toml), include the
ultimate source of the mapping into the .debug_info member of the
Package object. This makes it easier to debug the source of these
user-defined meppings.

In other words, given a mapping.toml with:

  apache-airflow = ["airflow"]

and a pyproject.toml with:

  [tool.fawltydeps.custom_mapping]
  apache-airflow = ["foo", "bar"]

we used to generate this Package object:

  Package(
    package_name="apache_airflow",
    import_names={"airflow", "foo", "bar"},
    resolved_with=UserDefinedMapping,
    debug_info=None,
  )

but with this commit we instead get:

  Package(
    package_name="apache_airflow",
    import_names={"airflow", "foo", "bar"},
    resolved_with=UserDefinedMapping,
    debug_info={
      "mapping.toml": {"airflow},
      "from settings": {"foo", "bar"},
    },
  )

Also refactor the accumulate_mappings() helper out of the
UserDefinedMapping class as we will soon start to use it from
LocalPackageResolver as well.

Author: jherland

fawltydeps/packages.py

@@ -8,9 +8,8 @@
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
 from dataclasses import dataclass, replace
-from itertools import chain
 from pathlib import Path
-from typing import Dict, Iterable, Iterator, List, Optional, Set, Type, Union
+from typing import Dict, Iterable, Iterator, List, Optional, Set, Tuple, Type, Union
 
 # importlib_metadata is gradually graduating into the importlib.metadata stdlib
 # module, however we rely on internal functions and recent (and upcoming)
@@ -85,6 +84,40 @@ def lookup_packages(self, package_names: Set[str]) -> Dict[str, Package]:
         raise NotImplementedError
 
 
+def accumulate_mappings(
+    resolved_with: Type[BasePackageResolver],
+    custom_mappings: Iterable[Tuple[CustomMapping, str]],
+) -> Dict[str, Package]:
+    """Merge CustomMappings (w/associated descriptions) into a dict of Packages.
+
+    Each resulting package object maps a (normalized) package name to a mapping
+    dict where the provided imports are keyed by their associated description.
+    The keys in the returned dict are also normalized package names.
+    """
+    result: Dict[str, Package] = {}
+    for custom_mapping, debug_key in custom_mappings:
+        for name, imports in custom_mapping.items():
+            normalized_name = Package.normalize_name(name)
+            if normalized_name not in result:  # create new Package instance
+                result[normalized_name] = Package(
+                    package_name=normalized_name,
+                    import_names=set(imports),
+                    resolved_with=resolved_with,
+                    debug_info={debug_key: set(imports)},
+                )
+            else:  # replace existing Package instance with "augmented" version
+                prev = result[normalized_name]
+                debug_info = prev.debug_info
+                assert isinstance(debug_info, dict)
+                debug_info.setdefault(debug_key, set()).update(imports)
+                result[normalized_name] = replace(
+                    prev,
+                    import_names=set.union(prev.import_names, imports),
+                    debug_info=debug_info,
+                )
+    return result
+
+
 class UserDefinedMapping(BasePackageResolver):
     """Use user-defined mapping loaded from a toml file"""
 
@@ -103,22 +136,6 @@ def __init__(
         # We enumerate packages declared in the mapping _once_ and cache the result here:
         self._packages: Optional[Dict[str, Package]] = None
 
-    @staticmethod
-    def accumulate_mappings(custom_mappings: Iterable[CustomMapping]) -> CustomMapping:
-        """Merge mapping dictionaries and normalise key (package) names."""
-        result: CustomMapping = {}
-        for name, imports in chain.from_iterable(cm.items() for cm in custom_mappings):
-            normalised_name = Package.normalize_name(name)
-            if normalised_name in result:
-                logger.info(
-                    "Mapping for %s already found. Import names "
-                    "from the second mapping are appended to ones "
-                    "found in the first mapping.",
-                    normalised_name,
-                )
-            result.setdefault(normalised_name, []).extend(imports)
-        return result
-
     @property
     @calculated_once
     def packages(self) -> Dict[str, Package]:
@@ -134,23 +151,18 @@ def packages(self) -> Dict[str, Package]:
         the remainder of this object's life in _packages.
         """
 
-        def _custom_mappings() -> Iterator[CustomMapping]:
+        def _custom_mappings() -> Iterator[Tuple[CustomMapping, str]]:
             if self.custom_mapping is not None:
                 logger.debug("Applying user-defined mapping from settings.")
-                yield self.custom_mapping
+                yield self.custom_mapping, "from settings"
 
             if self.mapping_paths is not None:
                 for path in self.mapping_paths:
                     logger.debug(f"Loading user-defined mapping from {path}")
                     with open(path, "rb") as mapping_file:
-                        yield tomllib.load(mapping_file)
+                        yield tomllib.load(mapping_file), str(path)
 
-        custom_mapping = self.accumulate_mappings(_custom_mappings())
-
-        return {
-            name: Package(name, set(imports), UserDefinedMapping)
-            for name, imports in custom_mapping.items()
-        }
+        return accumulate_mappings(self.__class__, _custom_mappings())
 
     def lookup_packages(self, package_names: Set[str]) -> Dict[str, Package]:
         """Convert package names to locally available Package objects."""

tests/test_packages.py

@@ -141,7 +141,12 @@ def test_package__local_env_mapping(
             """
             ],
             None,
-            {"apache_airflow": {"airflow"}, "attrs": {"attr", "attrs"}},
+            {
+                "apache_airflow": Package(
+                    "apache_airflow", {"airflow"}, UserDefinedMapping
+                ),
+                "attrs": Package("attrs", {"attr", "attrs"}, UserDefinedMapping),
+            },
             id="well_formated_input_file__parses_correctly",
         ),
         pytest.param(
@@ -157,9 +162,11 @@ def test_package__local_env_mapping(
             ],
             None,
             {
-                "apache_airflow": {"airflow", "baz"},
-                "attrs": {"attr", "attrs"},
-                "foo": {"bar"},
+                "apache_airflow": Package(
+                    "apache_airflow", {"airflow", "baz"}, UserDefinedMapping
+                ),
+                "attrs": Package("attrs", {"attr", "attrs"}, UserDefinedMapping),
+                "foo": Package("foo", {"bar"}, UserDefinedMapping),
             },
             id="well_formated_input_2files__parses_correctly",
         ),
@@ -176,9 +183,11 @@ def test_package__local_env_mapping(
             ],
             {"apache-airflow": ["unicorn"]},
             {
-                "apache_airflow": {"airflow", "baz", "unicorn"},
-                "attrs": {"attr", "attrs"},
-                "foo": {"bar"},
+                "apache_airflow": Package(
+                    "apache_airflow", {"airflow", "baz", "unicorn"}, UserDefinedMapping
+                ),
+                "attrs": Package("attrs", {"attr", "attrs"}, UserDefinedMapping),
+                "foo": Package("foo", {"bar"}, UserDefinedMapping),
             },
             id="well_formated_input_2files_and_config__parses_correctly",
         ),
@@ -199,8 +208,8 @@ def test_user_defined_mapping__well_formated_input_file__parses_correctly(
     udm = UserDefinedMapping(
         mapping_paths=custom_mapping_files, custom_mapping=custom_mapping
     )
-    mapped_packages = {k: v.import_names for k, v in udm.packages.items()}
-    assert mapped_packages == expect
+    actual = ignore_package_debug_info(udm.packages)
+    assert actual == expect
 
 
 def test_user_defined_mapping__input_is_no_file__raises_unparsable_path_exeption():

tests/test_resolver.py

@@ -151,14 +151,12 @@ def test_resolve_dependencies__generates_expected_mappings(
     )
 
     isolate_default_resolver(installed_deps)
-    actual = ignore_package_debug_info(
-        resolve_dependencies(
-            dep_names,
-            custom_mapping_files=set([custom_mapping_file])
-            if custom_mapping_file
-            else None,
-            custom_mapping=user_config_mapping,
-        )
+    actual = resolve_dependencies(
+        dep_names,
+        custom_mapping_files=set([custom_mapping_file])
+        if custom_mapping_file
+        else None,
+        custom_mapping=user_config_mapping,
     )
 
-    assert actual == expected
+    assert ignore_package_debug_info(actual) == ignore_package_debug_info(expected)