iiasa · khaeru · Nov 23, 2023 · Oct 30, 2023 · Oct 30, 2023 · Oct 30, 2023
diff --git a/.gitattributes b/.gitattributes
@@ -3,4 +3,5 @@ doc/whatsnew.rst merge=union
 # Git LFS
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.xlsx filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
@@ -62,6 +62,10 @@ jobs:
         version: 25.1.1
         license: ${{ secrets.GAMS_LICENSE }}
 
+    - uses: ts-graphviz/setup-graphviz@v1
+      with:
+        macos-skip-brew-update: true
+
     - name: Install packages and dependencies
       # By default, install:
       # - ixmp, message_ix: from GitHub branches/tags per matrix.upstream-version (above)
@@ -93,12 +97,6 @@ jobs:
           --local-cache --jvmargs="-Xmx6G"
       shell: bash
 
-    - name: Test documentation build using Sphinx
-      if: startsWith(matrix.version.os, 'ubuntu')
-      env:
-        RTD_TOKEN_MESSAGE_DATA: ${{ secrets.RTD_TOKEN_MESSAGE_DATA }}
-      run: make --directory=doc SPHINXOPTS="-n --color" html
-
     - name: Upload test coverage to Codecov.io
       uses: codecov/codecov-action@v3
 

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,2 +1,5 @@
+prune message_ix_models/data/test/advance
 prune message_ix_models/data/test/MESSAGEix-GLOBIOM_1.1_R11_no-policy_baseline
 prune message_ix_models/data/test/ssp
+prune message_ix_models/data/water/*
+exclude message_ix_models/data/water/*.tar.xz
diff --git a/doc/Makefile b/doc/Makefile
@@ -13,7 +13,7 @@ help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 clean-autosummary:
-	find . -name "$(BUILDDIR)" -prune -o -iname _autosummary -print0 | xargs -0 rmdir
+	find . -name "$(BUILDDIR)" -prune -o -iname _autosummary -print0 | xargs -0 rm -rf
 
 .PHONY: help clean-autosummary Makefile
 

diff --git a/doc/api/model-snapshot.rst b/doc/api/model-snapshot.rst
@@ -10,7 +10,7 @@ Usage
 
 From the command line, download data for a single snapshot::
 
-    $ mix-models snapshot fetch 0
+    $ mix-models fetch snapshot-0
 
 …where :program:`0` is the ID of a snapshot; see :data:`.SNAPSHOTS`.
 

diff --git a/doc/api/project.rst b/doc/api/project.rst
@@ -10,6 +10,23 @@ Specific research projects (:mod:`~message_ix_models.project`)
 .. automodule:: message_ix_models.project
    :members:
 
+.. currentmodule:: message_ix_models.project.advance
+
+ADVANCE (:mod:`.project.advance`)
+=================================
+
+.. automodule:: message_ix_models.project.advance
+
+.. automodule:: message_ix_models.project.advance.data
+   :members:
+
+   Although free of charge, the ADVANCE data can not be downloaded automatically.
+   This source requires that users first submit personal information to register before being able to retrieve the data.
+   :mod:`message_ix_models` does not circumvent this requirement.
+   Thus:
+
+   - A copy of the data are stored in :mod:`message_data`.
+   - :mod:`message_ix_models` contains only a ‘fuzzed’ version of the data (same structure, random values) for testing purposes.
 
 .. currentmodule:: message_ix_models.project.ssp
 
@@ -41,7 +58,7 @@ Data
 .. automodule:: message_ix_models.project.ssp.data
    :members:
 
-   Although free, neither the 2017 or 2024 SSP data can be downloaded automatically.
+   Although free of charge, neither the 2017 or 2024 SSP data can be downloaded automatically.
    Both sources require that users first submit personal information to register before being able to retrieve the data.
    :mod:`message_ix_models` does not circumvent this requirement.
    Thus:

diff --git a/doc/api/report/index.rst b/doc/api/report/index.rst
@@ -9,7 +9,7 @@ On this page:
 Elsewhere:
 
 - ``global.yaml``, the :doc:`default-config`.
-- Documentation for :mod:`genno` (:doc:`genno:index`), :mod:`ixmp.reporting`, and :mod:`message_ix.reporting`.
+- Documentation for :mod:`genno` (:doc:`genno:index`), :mod:`ixmp.report`, and :mod:`message_ix.report`.
 - Reporting of specific model variants:
 
   - :mod:`.water.reporting`
@@ -105,15 +105,16 @@ Plots
 .. automodule:: message_ix_models.report.plot
    :members:
 
-.. currentmodule:: message_ix_models.report.computations
+.. currentmodule:: message_ix_models.report.operator
 
 Operators
 ---------
 
-.. automodule:: message_ix_models.report.computations
+.. automodule:: message_ix_models.report.operator
    :members:
+   :exclude-members: add_par_data
 
-   :mod:`message_ix_models.report.computations` provides the following:
+   :mod:`message_ix_models.report.operator` provides the following:
 
    .. autosummary::
 
@@ -129,12 +130,18 @@ Operators
       remove_ts
       share_curtailment
 
+   The following functions, defined elsewhere, are exposed through :mod:`.operator` and so can also be referenced by name:
+
+   .. autosummary::
+
+      message_ix_models.util.add_par_data
+
    Other operators or genno-compatible functions are provided by:
 
    - Upstream packages:
 
-     - :mod:`message_ix.reporting.computations`
-     - :mod:`ixmp.reporting.computations`
+     - :mod:`message_ix.report.operator`
+     - :mod:`ixmp.report.operator`
      - :mod:`genno.computations`
 
    - Other submodules:

diff --git a/doc/api/tools.rst b/doc/api/tools.rst
@@ -61,6 +61,9 @@ Exogenous data (:mod:`.tools.exo_data`)
 ADVANCE data (:mod:`.tools.advance`)
 ====================================
 
+.. deprecated:: 2023.11
+   Use :mod:`.project.advance` instead.
+
 .. autosummary::
    get_advance_data
    advance_data
@@ -86,3 +89,11 @@ With :mod:`message_data`:
    :members:
    :exclude-members: LOCATION
    :private-members:
+
+.. currentmodule:: message_ix_models.tools.iamc
+
+IAMC data structures (:mod:`.tools.iamc`)
+=========================================
+
+.. automodule:: message_ix_models.tools.iamc
+   :members:
diff --git a/doc/api/util.rst b/doc/api/util.rst
@@ -36,6 +36,7 @@ Commonly used:
    copy_column
    ffill
    identify_nodes
+   iter_keys
    load_package_data
    load_private_data
    local_data_path
@@ -44,6 +45,7 @@ Commonly used:
    make_source_tech
    maybe_query
    merge_data
+   minimum_version
    ~node.nodes_ex_world
    package_data_path
    private_data_path

@@ -41,7 +41,7 @@ The generic pattern for workflows is:
 - The same precursor scenario can be used as the basis for multiple target scenarios.
 - A workflow is :meth:`.Workflow.run` starting with the earliest precursor scenario, ending with 1 or many target scenarios.
 
-The implementation is based on the observation that these form a graph (specifically, a directed, acyclic graph, or DAG) of nodes (= scenarios) and edges (= steps), in the same way that :mod:`message_ix.reporting` calculations do; and so the :mod:`dask` DAG features (via :mod:`genno`) can be used to organize the workflow.
+The implementation is based on the observation that these form a graph (specifically, a directed, acyclic graph, or DAG) of nodes (= scenarios) and edges (= steps), in the same way that :mod:`message_ix.report` calculations do; and so the :mod:`dask` DAG features (via :mod:`genno`) can be used to organize the workflow.
 
 Usage
 =====

diff --git a/doc/conf.py b/doc/conf.py
@@ -4,7 +4,8 @@
 # the documentation: https://www.sphinx-doc.org/en/master/usage/configuration.html
 
 import os
-from typing import TYPE_CHECKING
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
 
 if TYPE_CHECKING:
     import sphinx.application
@@ -21,6 +22,7 @@
 # Add any Sphinx extension module names here, as strings. They can be extensions coming
 # with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
+    # "ixmp.util.sphinx_linkcode_github",
     "sphinx.ext.autodoc",
     "sphinx.ext.autosummary",
     "sphinx.ext.extlinks",
@@ -39,6 +41,7 @@
 # html_extra_path.
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
+nitpicky = True
 nitpick_ignore_regex = {
     # These occur because there is no .. py:module:: directive for the *top-level*
     # module or package in the respective documentation and inventories.
@@ -109,18 +112,32 @@ def setup(app: "sphinx.application.Sphinx") -> None:
 
 # -- Options for sphinx.ext.intersphinx ------------------------------------------------
 
+
+def local_inv(name: str, *parts: str) -> Optional[str]:
+    """Construct the path to a local intersphinx inventory."""
+    if 0 == len(parts):
+        parts = ("doc", "_build", "html")
+
+    from importlib.util import find_spec
+
+    spec = find_spec(name)
+    if spec and spec.origin:
+        return str(Path(spec.origin).parents[1].joinpath(*parts, "objects.inv"))
+
+
 # For message-data, see: https://docs.readthedocs.io/en/stable/guides
 # /intersphinx.html#intersphinx-with-private-projects
 _token = os.environ.get("RTD_TOKEN_MESSAGE_DATA", "")
 
 intersphinx_mapping = {
+    "click": ("https://click.palletsprojects.com/en/8.1.x/", None),
     "genno": ("https://genno.readthedocs.io/en/stable", None),
     "ixmp": ("https://docs.messageix.org/projects/ixmp/en/latest/", None),
     "message-ix": ("https://docs.messageix.org/en/latest/", None),
     "m-data": (
         f"https://{_token}:@docs.messageix.org/projects/models-internal/en/latest/",
         # Use a local copy of objects.inv, if the user has one
-        (None, "message_data.inv"),
+        (local_inv("message_data"), None),
     ),
     "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
     "pint": ("https://pint.readthedocs.io/en/stable/", None),
@@ -130,8 +147,13 @@ def setup(app: "sphinx.application.Sphinx") -> None:
     "sdmx": ("https://sdmx1.readthedocs.io/en/stable/", None),
 }
 
+# -- Options for sphinx.ext.linkcode / ixmp.util.sphinx_linkcode_github ----------------
+
+linkcode_github_repo_slug = "iiasa/message-ix-models"
+
 # -- Options for sphinx.ext.napoleon ---------------------------------------------------
 
+napoleon_preprocess_types = True
 napoleon_type_aliases = {
     "Code": ":class:`~sdmx.model.common.Code`",
     "Path": ":class:`~pathlib.Path`",

diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst
@@ -4,7 +4,45 @@ What's new
 Next release
 ============
 
+Migration notes
+---------------
+Update code that imports from the following modules:
+
+- :py:`message_ix_models.report.computations` → use :py:`message_ix_models.report.operator`.
+
+Code that imports from the old locations will continue to work, but will raise :class:`DeprecationWarning`.
+
+Data for :doc:`water` is no longer included in the PyPI distributions for :mod:`message_ix_models`.
+This reduces the package size from >20 MB to <5 MB.
+To automatically download and unpack these data into a local directory, use :program:`mix-models fetch MESSAGEix-Nexus`.
+
+All changes
+-----------
+
+- Improve :class:`.ExoDataSource` (:pull:`137`):
+
+  - New attributes :attr:`~.ExoDataSource.name`, :attr:`~.ExoDataSource.extra_dims`.
+  - New method :meth:`~.ExoDataSource.transform` that can be overridden by subclasses.
+  - New arguments :py:`archive_member`, :py:`non_iso_3166` to :func:`.iamc_like_data_for_query`.
+
+- New provider for exogenous data from the :class:`.ADVANCE` project (:pull:`137`).
+  This module, :mod:`.project.advance`, supersedes :mod:`.tools.advance` and its idiosyncratic API, which are deprecated.
+- New CLI commands (:pull:`137`):
+
+  - :program:`mix-models testing fuzz-private-data`, superseding :program:`mix-models ssp make-test-data`.
+  - :program:`mix-models fetch`, superseding :program:`mix-models snapshot fetch`.
+
+- New utility functions  (:pull:`137`).
+
+  - :func:`.tools.iamc.describe` to generate SDMX code lists that describe the structure of particular IAMC-format data (:pull:`137`).
+  - :func:`.workflow.make_click_command` to generate :mod:`click` commands for any :class:`.Workflow`.
+  - :func:`.util.minimum_version` to ensure compatibility with upstream packages and aid test writing.
+  - :func:`.util.iter_keys` to generate keys for chains of :mod:`genno` computations.
+
 - Add :mod:`message_ix_models.report.compat` :ref:`for emulating legacy reporting <report-legacy>` (:pull:`134`).
+- Rename :mod:`message_ix_models.report.operator` (:pull:`137`).
+- Deprecate :py:`iter_parameters()` in favour of :meth:`ixmp.Scenario.par_list` with :py:`indexed_by=...` argument from ixmp v3.8.0 (:pull:`137`).
+
 
 v2023.10.16
 ===========
@@ -218,7 +256,7 @@ Earlier releases
 ---------
 
 - Migrate :mod:`.model.bare`, :mod:`.model.build`, :mod:`.model.cli`, and associated documentation (:pull:`9`)
-- Migrate utilities: :class:`.ScenarioInfo`, :func:`.add_par_data`, :func:`.eval_anno`, :func:`.iter_parameters`, and :func:`.strip_par_data`.
+- Migrate utilities: :class:`.ScenarioInfo`, :func:`.add_par_data`, :func:`.eval_anno`, :py:`iter_parameters()`, and :func:`.strip_par_data`.
 
 2021.3.3
 --------

diff --git a/message_ix_models/cli.py b/message_ix_models/cli.py
@@ -111,11 +111,12 @@ def debug(ctx):
 #: or @click.group.
 submodules = [
     "message_ix_models.model.cli",
-    "message_ix_models.model.snapshot",
     "message_ix_models.model.structure",
     "message_ix_models.model.water.cli",
     "message_ix_models.project.ssp",
     "message_ix_models.report.cli",
+    "message_ix_models.testing.cli",
+    "message_ix_models.util.pooch",
 ]
 
 try:

diff --git a/message_ix_models/data/report/global.yaml b/message_ix_models/data/report/global.yaml
@@ -41,7 +41,7 @@ units:
 
 # Filters
 #
-# These limit the data that is retrieved from the backend by ixmp.reporting;
+# These limit the data that is retrieved from the backend by ixmp.report;
 # so ALL quantities in the Reporter are limited to these values. Use these for
 # debugging.