Merge 3.28.x into main (#5121)

.authors.yml

@@ -1201,7 +1201,7 @@
   alternate_emails:
   - clee@anaconda.com
 - name: Ken Odegard
-  num_commits: 152
+  num_commits: 155
   email: kodegard@anaconda.com
   first_commit: 2020-09-08 19:53:41
   github: kenodegard
@@ -1326,7 +1326,7 @@
 - name: Jaime Rodríguez-Guerra
   email: jaimergp@users.noreply.github.com
   github: jaimergp
-  num_commits: 7
+  num_commits: 8
   first_commit: 2022-11-02 19:34:51
 - name: Dave Clements
   email: tnabtaf@gmail.com

CHANGELOG.md

@@ -1,5 +1,25 @@
 [//]: # (current developments)
 
+## 3.28.2 (2023-12-15)
+
+### Enhancements
+
+* Update `conda_build.metadata.MetaData.get_section` to consistently return lists for "source" and "outputs". (#5111 via #5112)
+
+### Bug fixes
+
+* Resolve duplicate package record issue in `conda_build.inspect_pkg.which_package`. (#5106 via #5108)
+* Ensure `conda_build.post._lookup_in_prefix_packages` displays `str(PackageRecord)` instead of `repr(PackageRecord)`. (#5106 via #5108)
+* Fix finalization of recipes with multiple sources. (#5111 via #5112)
+* Improve handling by catching the more general `ImportError` instead of `ModuleNotFoundError` to cover cases involving `menuinst 1.x`. (#5116)
+
+### Contributors
+
+* @jaimergp
+* @kenodegard
+
+
+
 ## 3.28.1 (2023-12-06)
 
 ### Bug fixes

conda_build/build.py

@@ -1564,15 +1564,11 @@ def create_info_files(m, replacements, files, prefix):
 
     write_no_link(m, files)
 
-    sources = m.get_section("source")
-    if hasattr(sources, "keys"):
-        sources = [sources]
-
     with open(join(m.config.info_dir, "git"), "w", encoding="utf-8") as fo:
-        for src in sources:
-            if src.get("git_url"):
+        for source_dict in m.get_section("source"):
+            if source_dict.get("git_url"):
                 source.git_info(
-                    os.path.join(m.config.work_dir, src.get("folder", "")),
+                    os.path.join(m.config.work_dir, source_dict.get("folder", "")),
                     m.config.build_prefix,
                     git=None,
                     verbose=m.config.verbose,

conda_build/inspect_pkg.py

@@ -73,9 +73,8 @@ def which_package(
     path = prefix / path
 
     for prec in PrefixData(str(prefix)).iter_records():
-        for file in prec["files"]:
-            if samefile(prefix / file, path):
-                yield prec
+        if any(samefile(prefix / file, path) for file in prec["files"]):
+            yield prec
 
 
 def print_object_info(info, key):

conda_build/metadata.py

@@ -13,7 +13,7 @@
 from collections import OrderedDict
 from functools import lru_cache
 from os.path import isfile, join
-from typing import Literal
+from typing import Literal, overload
 
 from bs4 import UnicodeDammit
 
@@ -622,6 +622,7 @@ def parse(data, config, path=None):
         "prelink_message": None,
         "readme": None,
     },
+    "extra": {},
 }
 
 # Fields that may either be a dictionary or a list of dictionaries.
@@ -1316,9 +1317,11 @@ def parse_until_resolved(
     @classmethod
     def fromstring(cls, metadata, config=None, variant=None):
         m = super().__new__(cls)
-        if not config:
-            config = Config()
-        m.meta = parse(metadata, config=config, path="", variant=variant)
+        m.path = ""
+        m._meta_path = ""
+        m.requirements_path = ""
+        config = config or Config(variant=variant)
+        m.meta = parse(metadata, config=config, path="")
         m.config = config
         m.parse_again(permit_undefined_jinja=True)
         return m
@@ -1333,18 +1336,45 @@ def fromdict(cls, metadata, config=None, variant=None):
         m._meta_path = ""
         m.requirements_path = ""
         m.meta = sanitize(metadata)
-
-        if not config:
-            config = Config(variant=variant)
-
-        m.config = config
+        m.config = config or Config(variant=variant)
         m.undefined_jinja_vars = []
         m.final = False
-
         return m
 
-    def get_section(self, section):
-        return self.meta.get(section, {})
+    @overload
+    def get_section(self, section: Literal["source", "outputs"]) -> list[dict]:
+        ...
+
+    @overload
+    def get_section(
+        self,
+        section: Literal[
+            "package",
+            "build",
+            "requirements",
+            "app",
+            "test",
+            "about",
+            "extra",
+        ],
+    ) -> dict:
+        ...
+
+    def get_section(self, name):
+        section = self.meta.get(name)
+        if name in OPTIONALLY_ITERABLE_FIELDS:
+            if not section:
+                return []
+            elif isinstance(section, dict):
+                return [section]
+            elif not isinstance(section, list):
+                raise ValueError(f"Expected {name} to be a list")
+        else:
+            if not section:
+                return {}
+            elif not isinstance(section, dict):
+                raise ValueError(f"Expected {name} to be a dict")
+        return section
 
     def get_value(self, name, default=None, autotype=True):
         """
@@ -1364,7 +1394,9 @@ def get_value(self, name, default=None, autotype=True):
             index = None
         elif len(names) == 3:
             section, index, key = names
-            assert section == "source", "Section is not a list: " + section
+            assert section in OPTIONALLY_ITERABLE_FIELDS, (
+                "Section is not a list: " + section
+            )
             index = int(index)
 
         # get correct default
@@ -1386,7 +1418,7 @@ def get_value(self, name, default=None, autotype=True):
                 )
                 index = 0
 
-            if len(section_data) == 0:
+            if not section_data:
                 section_data = {}
             else:
                 section_data = section_data[index]
@@ -1475,7 +1507,7 @@ def get_depends_top_and_out(self, typ):
         if not self.is_output:
             matching_output = [
                 out
-                for out in self.meta.get("outputs", [])
+                for out in self.get_section("outputs")
                 if out.get("name") == self.name()
             ]
             if matching_output:
@@ -2014,7 +2046,7 @@ def uses_jinja(self):
         return len(matches) > 0
 
     @property
-    def uses_vcs_in_meta(self) -> Literal["git" | "svn" | "mercurial"] | None:
+    def uses_vcs_in_meta(self) -> Literal["git", "svn", "mercurial"] | None:
         """returns name of vcs used if recipe contains metadata associated with version control systems.
         If this metadata is present, a download/copy will be forced in parse_or_try_download.
         """
@@ -2034,7 +2066,7 @@ def uses_vcs_in_meta(self) -> Literal["git" | "svn" | "mercurial"] | None:
         return vcs
 
     @property
-    def uses_vcs_in_build(self) -> Literal["git" | "svn" | "mercurial"] | None:
+    def uses_vcs_in_build(self) -> Literal["git", "svn", "mercurial"] | None:
         # TODO :: Re-work this. Is it even useful? We can declare any vcs in our build deps.
         build_script = "bld.bat" if on_win else "build.sh"
         build_script = os.path.join(self.path, build_script)
@@ -2271,9 +2303,8 @@ def pin_depends(self):
 
     @property
     def source_provided(self):
-        return not bool(self.meta.get("source")) or (
-            os.path.isdir(self.config.work_dir)
-            and len(os.listdir(self.config.work_dir)) > 0
+        return not self.get_section("source") or (
+            os.path.isdir(self.config.work_dir) and os.listdir(self.config.work_dir)
         )
 
     def reconcile_metadata_with_output_dict(self, output_metadata, output_dict):

conda_build/post.py

@@ -1187,9 +1187,7 @@ def _lookup_in_prefix_packages(
     if len(precs_in_reqs) == 1:
         _print_msg(
             errors,
-            "{}: {} found in {}{}".format(
-                info_prelude, n_dso_p, precs_in_reqs[0], and_also
-            ),
+            f"{info_prelude}: {n_dso_p} found in {precs_in_reqs[0]}{and_also}",
             verbose=verbose,
         )
     elif in_whitelist:
@@ -1201,25 +1199,20 @@ def _lookup_in_prefix_packages(
     elif len(precs_in_reqs) == 0 and len(precs) > 0:
         _print_msg(
             errors,
-            "{}: {} found in {}{}".format(
-                msg_prelude, n_dso_p, [prec.name for prec in precs], and_also
-            ),
+            f"{msg_prelude}: {n_dso_p} found in {[str(prec) for prec in precs]}{and_also}",
             verbose=verbose,
         )
         _print_msg(
             errors,
-            "{}: .. but {} not in reqs/run, (i.e. it is overlinking)"
-            " (likely) or a missing dependency (less likely)".format(
-                msg_prelude, [prec.name for prec in precs]
-            ),
+            f"{msg_prelude}: .. but {[str(prec) for prec in precs]} not in reqs/run, "
+            "(i.e. it is overlinking) (likely) or a missing dependency (less likely)",
             verbose=verbose,
         )
     elif len(precs_in_reqs) > 1:
         _print_msg(
             errors,
-            "{}: {} found in multiple packages in run/reqs: {}{}".format(
-                warn_prelude, in_prefix_dso, precs_in_reqs, and_also
-            ),
+            f"{warn_prelude}: {in_prefix_dso} found in multiple packages in run/reqs: "
+            f"{[str(prec) for prec in precs_in_reqs]}{and_also}",
             verbose=verbose,
         )
     else:
@@ -1753,7 +1746,7 @@ def check_menuinst_json(files, prefix) -> None:
     try:
         import jsonschema
         from menuinst.utils import data_path
-    except ModuleNotFoundError as exc:
+    except ImportError as exc:
         log.warning(
             "Found 'Menu/*.json' files but couldn't validate: %s",
             ", ".join(json_files),

conda_build/render.py

@@ -721,17 +721,17 @@ def finalize_metadata(
         # if source/path is relative, then the output package makes no sense at all.  The next
         #   best thing is to hard-code the absolute path.  This probably won't exist on any
         #   system other than the original build machine, but at least it will work there.
-        if source_path := m.get_value("source/path"):
-            if not isabs(source_path):
-                m.meta["source"]["path"] = normpath(join(m.path, source_path))
+        for source_dict in m.get_section("source"):
+            if (source_path := source_dict.get("path")) and not isabs(source_path):
+                source_dict["path"] = normpath(join(m.path, source_path))
             elif (
-                (git_url := m.get_value("source/git_url"))
+                (git_url := source_dict.get("git_url"))
                 # absolute paths are not relative paths
                 and not isabs(git_url)
                 # real urls are not relative paths
                 and ":" not in git_url
             ):
-                m.meta["source"]["git_url"] = normpath(join(m.path, git_url))
+                source_dict["git_url"] = normpath(join(m.path, git_url))
 
         m.meta.setdefault("build", {})
 

conda_build/source.py

@@ -1027,18 +1027,11 @@ def provide(metadata):
       - unpack
       - apply patches (if any)
     """
-    meta = metadata.get_section("source")
-    if not os.path.isdir(metadata.config.build_folder):
-        os.makedirs(metadata.config.build_folder)
+    os.makedirs(metadata.config.build_folder, exist_ok=True)
     git = None
 
-    if hasattr(meta, "keys"):
-        dicts = [meta]
-    else:
-        dicts = meta
-
     try:
-        for source_dict in dicts:
+        for source_dict in metadata.get_section("source"):
             folder = source_dict.get("folder")
             src_dir = os.path.join(metadata.config.work_dir, folder if folder else "")
             if any(k in source_dict for k in ("fn", "url")):