Add #137 and contents to doc/whatsnew

doc/api/project.rst

@@ -15,10 +15,12 @@ Specific research projects (:mod:`~message_ix_models.project`)
 ADVANCE (:mod:`.project.advance`)
 =================================
 
+.. automodule:: message_ix_models.project.advance
+
 .. automodule:: message_ix_models.project.advance.data
    :members:
 
-   Although free, the ADVANCE data can not be downloaded automatically.
+   Although free of charge, the ADVANCE data can not be downloaded automatically.
    This source requires that users first submit personal information to register before being able to retrieve the data.
    :mod:`message_ix_models` does not circumvent this requirement.
    Thus:
@@ -56,7 +58,7 @@ Data
 .. automodule:: message_ix_models.project.ssp.data
    :members:
 
-   Although free, neither the 2017 or 2024 SSP data can be downloaded automatically.
+   Although free of charge, neither the 2017 or 2024 SSP data can be downloaded automatically.
    Both sources require that users first submit personal information to register before being able to retrieve the data.
    :mod:`message_ix_models` does not circumvent this requirement.
    Thus:

doc/api/tools.rst

@@ -61,6 +61,9 @@ Exogenous data (:mod:`.tools.exo_data`)
 ADVANCE data (:mod:`.tools.advance`)
 ====================================
 
+.. deprecated:: 2023.11
+   Use :mod:`.project.advance` instead.
+
 .. autosummary::
    get_advance_data
    advance_data
@@ -86,3 +89,11 @@ With :mod:`message_data`:
    :members:
    :exclude-members: LOCATION
    :private-members:
+
+.. currentmodule:: message_ix_models.tools.iamc
+
+IAMC data structures (:mod:`.tools.iamc`)
+=========================================
+
+.. automodule:: message_ix_models.tools.iamc
+   :members:

doc/api/util.rst

@@ -36,6 +36,7 @@ Commonly used:
    copy_column
    ffill
    identify_nodes
+   iter_keys
    load_package_data
    load_private_data
    local_data_path

doc/conf.py

@@ -115,23 +115,22 @@ def setup(app: "sphinx.application.Sphinx") -> None:
 
 def local_inv(name: str, *parts: str) -> Optional[str]:
     """Construct the path to a local intersphinx inventory."""
+    if 0 == len(parts):
+        parts = ("doc", "_build", "html")
 
     from importlib.util import find_spec
 
     spec = find_spec(name)
-    if spec is None:
-        return None
-
-    if 0 == len(parts):
-        parts = ("doc", "_build", "html")
-    return str(Path(spec.origin).parents[1].joinpath(*parts, "objects.inv"))
+    if spec and spec.origin:
+        return str(Path(spec.origin).parents[1].joinpath(*parts, "objects.inv"))
 
 
 # For message-data, see: https://docs.readthedocs.io/en/stable/guides
 # /intersphinx.html#intersphinx-with-private-projects
 _token = os.environ.get("RTD_TOKEN_MESSAGE_DATA", "")
 
 intersphinx_mapping = {
+    "click": ("https://click.palletsprojects.com/en/8.1.x/", None),
     "genno": ("https://genno.readthedocs.io/en/stable", None),
     "ixmp": ("https://docs.messageix.org/projects/ixmp/en/latest/", None),
     "message-ix": ("https://docs.messageix.org/en/latest/", None),

doc/whatsnew.rst

@@ -4,9 +4,45 @@ What's new
 Next release
 ============
 
-- Deprecate :py:`iter_parameters()` in favour of :meth:`.Scenario.par_list` with :py:`indexed_by=...` argument from ixmp v3.8.0 (:pull:`137`).
-- New utility method :func:`.minimum_version` for compatibility and testing (:pull:`137`).
+Migration notes
+---------------
+Update code that imports from the following modules:
+
+- :py:`message_ix_models.report.computations` → use :py:`message_ix_models.report.operator`.
+
+Code that imports from the old locations will continue to work, but will raise :class:`DeprecationWarning`.
+
+Data for :doc:`water` is no longer included in the PyPI distributions for :mod:`message_ix_models`.
+This reduces the package size from >20 MB to <5 MB.
+To automatically download and unpack these data into a local directory, use :program:`mix-models fetch MESSAGEix-Nexus`.
+
+All changes
+-----------
+
+- Improve :class:`.ExoDataSource` (:pull:`137`):
+
+  - New attributes :attr:`~.ExoDataSource.name`, :attr:`~.ExoDataSource.extra_dims`.
+  - New method :meth:`~.ExoDataSource.transform` that can be overridden by subclasses.
+  - New arguments :py:`archive_member`, :py:`non_iso_3166` to :func:`.iamc_like_data_for_query`.
+
+- New provider for exogenous data from the :class:`.ADVANCE` project (:pull:`137`).
+  This module, :mod:`.project.advance`, supersedes :mod:`.tools.advance` and its idiosyncratic API, which are deprecated.
+- New CLI commands (:pull:`137`):
+
+  - :program:`mix-models testing fuzz-private-data`, superseding :program:`mix-models ssp make-test-data`.
+  - :program:`mix-models fetch`, superseding :program:`mix-models snapshot fetch`.
+
+- New utility functions  (:pull:`137`).
+
+  - :func:`.tools.iamc.describe` to generate SDMX code lists that describe the structure of particular IAMC-format data (:pull:`137`).
+  - :func:`.workflow.make_click_command` to generate :mod:`click` commands for any :class:`.Workflow`.
+  - :func:`.util.minimum_version` to ensure compatibility with upstream packages and aid test writing.
+  - :func:`.util.iter_keys` to generate keys for chains of :mod:`genno` computations.
+
 - Add :mod:`message_ix_models.report.compat` :ref:`for emulating legacy reporting <report-legacy>` (:pull:`134`).
+- Rename :mod:`message_ix_models.report.operator` (:pull:`137`).
+- Deprecate :py:`iter_parameters()` in favour of :meth:`ixmp.Scenario.par_list` with :py:`indexed_by=...` argument from ixmp v3.8.0 (:pull:`137`).
+
 
 v2023.10.16
 ===========

message_ix_models/project/advance/data.py

@@ -140,6 +140,11 @@ def __call__(self):
         )
 
     def transform(self, c: "Computer", base_key: Key) -> Key:
+        """Prepare `c` to transform raw data from `base_key`.
+
+        Unlike the base class version, this implementation only adds the aggregation
+        step if :attr:`.aggregate` is :any:`True`.
+        """
         k = iter_keys(base_key)
 
         k1 = base_key

message_ix_models/tools/advance.py

@@ -20,18 +20,19 @@
 
 #: Standard dimensions for data produced as snapshots from the IIASA ENE Program
 #: “WorkDB”.
-#:
-#: .. todo:: Move to a common location for use with other snapshots in the same format.
 DIMS = ["model", "scenario", "region", "variable", "unit", "year"]
 
 
 @cached
 def get_advance_data(query: Optional[str] = None) -> pd.Series:
     """Return data from the ADVANCE Work Package 2 data snapshot at :data:`LOCATION`.
 
+    .. deprecated:: 2023.11
+       Use :class:`.ADVANCE` through :func:`.exo_data.prepare_computer` instead.
+
     Parameters
     ----------
-    query : str, *optional*
+    query : str, optional
         Passed to :meth:`pandas.DataFrame.query` to limit the returned values.
 
     Returns
@@ -50,9 +51,12 @@ def get_advance_data(query: Optional[str] = None) -> pd.Series:
 def advance_data(variable: str, query: Optional[str] = None) -> Quantity:
     """Return a single ADVANCE data `variable` as a :class:`genno.Quantity`.
 
+    .. deprecated:: 2023.11
+       Use :class:`.ADVANCE` through :func:`.exo_data.prepare_computer` instead.
+
     Parameters
     ----------
-    query : str, *optional*
+    query : str, optional
         Passed to :func:`get_advance_data`.
 
     Returns
@@ -95,7 +99,8 @@ def _read_workdb_snapshot(path: Path, name: str) -> pd.Series:
     the “IAMC format,” without more recent additions intended to represent sub-annual
     time resolution using a separate column.)
 
-    .. todo:: Move to a general location for use with other files in the same format.
+    .. deprecated:: 2023.11
+       Use :func:`.iamc_like_data_for_query` instead.
     """
     with ZipFile(path) as zf:  # Open the ZIP archive
         with zf.open(name) as f:  # Open a particular member

message_ix_models/tools/exo_data.py

@@ -301,6 +301,15 @@ def iamc_like_data_for_query(
     7. Transform to :class:`.Quantity` with units.
 
     The result is :obj:`.cached`.
+
+    Parameters
+    ----------
+    archive_member : bool, optional
+        If given, `path` may be an archive with 2 or more members. The member named by
+        `archive_member` is extracted and read.
+    non_iso_3166 : bool, optional
+        If "discard" (default), "region" labels that are not ISO 3166-1 country names
+        are discarded, along with associated data. If "keep", such labels are kept.
     """
     import pandas as pd
 

message_ix_models/tools/iamc.py

@@ -6,8 +6,24 @@
 from sdmx.message import StructureMessage
 
 
-def describe(data: pd.DataFrame, extra: Optional[str] = None):
-    """Generate code lists from `data` in IAMC format."""
+def describe(data: pd.DataFrame, extra: Optional[str] = None) -> StructureMessage:
+    """Generate SDMX structure information from `data` in IAMC format.
+
+    Parameters
+    ----------
+    data :
+        Data in "wide" or "long" IAMC format.
+    extra : str, optional
+        Extra text added to the description of each Codelist.
+
+    Returns
+    -------
+    sdmx.message.StructureMessage
+        The message contains one :class:`.Codelist` for each of the MODEL, SCENARIO,
+        REGION, VARIABLE, and UNIT dimensions. Codes for the VARIABLE code list have
+        annotations with :py:`id="preferred-unit-measure"` that give the corresponding
+        UNIT Code(s) that appear with each VARIABLE.
+    """
 
     sm = StructureMessage()
 

message_ix_models/util/__init__.py

@@ -326,6 +326,22 @@ def __call__(self) -> "genno.Key":
 
 
 def iter_keys(base: "genno.Key") -> KeyIterator:
+    """Return an iterator over a sequence of keys starting with `base_key`.
+
+    This can be used for shorthand when constructing sequences of :mod:`genno`
+    computations.
+
+    Example
+    -------
+    >>> base_key = genno.Key("foo:a-b-c")
+    >>> k = iter_keys(base_key)
+    >>> k()
+    <foo:a-b-c:0>
+    >>> k()
+    <foo:a-b-c:1>
+    >>> k()
+    <foo:a-b-c:2>
+    """
     return partial(next, map(lambda i: base + str(i), count()))