Merge pull request #1251 from gboeing/envs

generate conda env yml and pip requirements files automatically using script

docs/.readthedocs.yaml

@@ -9,7 +9,7 @@ formats: all
 
 python:
   install:
-    - requirements: docs/requirements.txt
+    - requirements: docs/requirements-rtd.txt
 
 sphinx:
   configuration: docs/source/conf.py

docs/requirements-rtd.txt

@@ -0,0 +1,6 @@
+# Do not edit this file. It is automatically generated by the script
+# environments/make-env-files.py using the environment definition data in
+# environments/environments.json and the requirements in pyproject.toml.
+furo
+sphinx-autodoc-typehints
+sphinx>=7

docs/source/conf.py

@@ -63,7 +63,7 @@
 html_static_path: list[str] = []
 html_theme = "furo"
 language = "en"
-needs_sphinx = "7"  # same value as pinned in /docs/requirements.txt
+needs_sphinx = "7"  # match version from pyproject.toml optional-dependencies
 root_doc = "index"
 source_suffix = ".rst"
 templates_path: list[str] = []

environments/docker/build-image.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -euo pipefail
 echo "Run this script from the repository root."
 docker login
 docker buildx build --no-cache --pull --push --platform=linux/amd64,linux/arm64 -f ./environments/docker/Dockerfile -t gboeing/osmnx:test .

environments/environments.json

@@ -0,0 +1,65 @@
+{
+ "env-test-minimum-deps": {
+  "filepath": "./environments/tests/env-test-minimum-deps.yml",
+  "needs_python": true,
+  "needs_dependencies": true,
+  "needs_optionals": true,
+  "which_optionals": ["entropy", "neighbors", "raster", "visualization", "tests"],
+  "force_pin": true,
+  "extras": null
+ },
+ "env-ci": {
+  "filepath": "./environments/tests/env-ci.yml",
+  "needs_python": true,
+  "needs_dependencies": true,
+  "needs_optionals": true,
+  "which_optionals": null,
+  "force_pin": false,
+  "extras": null
+ },
+ "requirements-test-latest-deps": {
+  "filepath": "./environments/tests/requirements-test-latest-deps.txt",
+  "needs_python": false,
+  "needs_dependencies": true,
+  "needs_optionals": true,
+  "which_optionals": null,
+  "force_pin": false,
+  "extras": null
+ },
+ "requirements-rtd": {
+  "filepath": "./docs/requirements-rtd.txt",
+  "needs_python": false,
+  "needs_dependencies": false,
+  "needs_optionals": true,
+  "which_optionals": ["docs"],
+  "force_pin": false,
+  "extras": null
+ },
+ "requirements-env": {
+  "filepath": "./environments/requirements.txt",
+  "needs_python": false,
+  "needs_dependencies": true,
+  "needs_optionals": true,
+  "which_optionals": null,
+  "force_pin": false,
+  "extras": [
+   "bottleneck",
+   "cartopy",
+   "conda-smithy",
+   "folium",
+   "hatch",
+   "jupyterlab",
+   "nbdime",
+   "nbqa",
+   "numexpr",
+   "pillow",
+   "pip",
+   "pysal>24",
+   "python-igraph",
+   "seaborn",
+   "statsmodels",
+   "twine",
+   "validate-pyproject"
+  ]
+ }
+}

environments/make-env-files.py

@@ -0,0 +1,155 @@
+# noqa: INP001
+"""Make conda env.yml and pip requirements.txt files from environments.json data."""
+
+from __future__ import annotations
+
+import argparse
+import itertools
+import json
+from pathlib import Path
+
+import tomllib
+from packaging.requirements import Requirement
+
+# path to package's pyproject and the config json file
+pyproject_path = "./pyproject.toml"
+environments_config_path = "./environments/environments.json"
+
+# what channels to specify in conda env yml files
+CHANNELS = ["conda-forge"]
+
+HEADER = (
+    "# Do not edit this file. It is automatically generated by the script\n"
+    "# environments/make-env-files.py using the environment definition data in\n"
+    "# environments/environments.json and the requirements in pyproject.toml.\n"
+)
+
+
+def extract_optional_deps(which: list[str] | None = None) -> list[Requirement]:
+    """
+    Extract a list of the optional dependencies/versions from pyproject.toml.
+
+    Parameters
+    ----------
+    which
+        Which optional dependencies to extract. If None, extract them all.
+
+    Returns
+    -------
+    optional_deps
+    """
+    opts = pyproject["project"]["optional-dependencies"]
+    opts = [v for k, v in opts.items() if k in which] if which is not None else opts.values()
+    return list({Requirement(o) for o in itertools.chain.from_iterable(opts)})
+
+
+def make_requirement(
+    requirement: Requirement,
+    force_pin: bool = False,  # noqa: FBT001,FBT002
+    is_conda: bool = True,  # noqa: FBT001,FBT002
+) -> str:
+    """
+    Make a requirement specification string.
+
+    The string result comprises the requirement's name and its specifier(s).
+
+    Parameters
+    ----------
+    requirement
+        A requirement object
+    force_pin
+        If True, pin requirement to version rather than using existing
+        specifier. Allows you to convert minimum versions to pinned versions.
+    is_conda
+        If True and if `force_pin` is True, format the requirement string to
+        end with ".*" for conda environment file pinning format compatibility.
+
+    Returns
+    -------
+    requirement_str
+    """
+    specifiers = list(requirement.specifier)
+    if force_pin and len(specifiers) == 1:
+        spec = f"{requirement.name}=={specifiers[0].version}"
+        if is_conda and not spec.endswith(".*"):
+            spec += ".*"
+        return spec
+    return str(requirement)
+
+
+def make_file(env_name: str) -> None:
+    """
+    Write a conda environment yaml file or pip requirements.txt file.
+
+    Parameters
+    ----------
+    env_name
+        An enviroment name among the keys of environments.json.
+
+    Returns
+    -------
+    None
+    """
+    env = envs[env_name]
+
+    # it's a conda env file if it ends with ".yml", otherwise it's a pip
+    # requirements.txt file
+    is_conda = env["filepath"].endswith(".yml")
+
+    # determine which dependencies to add based on the configuration
+    depends_on = []
+    if env["needs_python"]:
+        python_dep = Requirement(f"python{pyproject['project']['requires-python']}")
+        depends_on.append(python_dep)
+    if env["needs_dependencies"]:
+        dependencies = [Requirement(d) for d in pyproject["project"]["dependencies"]]
+        depends_on.extend(dependencies)
+    if env["needs_optionals"]:
+        optionals = extract_optional_deps(which=env["which_optionals"])
+        depends_on.extend(optionals)
+
+    # make the list of requirements
+    requirements = sorted(
+        make_requirement(dep, force_pin=env["force_pin"], is_conda=is_conda) for dep in depends_on
+    )
+
+    # add any extra requirements if provided in the configuration
+    if env["extras"] is not None:
+        requirements = sorted(requirements + env["extras"])
+
+    # write the conda env yml or pip requirements.txt file to disk
+    with Path(env["filepath"]).open("w") as f:
+        if is_conda:
+            data = {"name": env_name, "channels": CHANNELS, "dependencies": requirements}
+            text = ""
+            for k, v in data.items():
+                if isinstance(v, list):
+                    text += k + ":\n- " + "\n- ".join(v) + "\n"
+                elif isinstance(v, str):
+                    text += k + ": " + v + "\n"
+            f.writelines(HEADER + text)
+        else:
+            f.writelines(HEADER + "\n".join(requirements) + "\n")
+
+    print(f"Wrote {len(requirements)} requirements to {env['filepath']!r}")  # noqa: T201
+
+
+if __name__ == "__main__":
+    # load the pyproject.toml and the environments.json config files
+    with Path(pyproject_path).open("rb") as f:
+        pyproject = tomllib.load(f)
+    with Path(environments_config_path).open("rb") as f:
+        envs = json.load(f)
+
+    # parse any command-line arguments passed by the user
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument("-n", dest="env_name", type=str)
+    args = arg_parser.parse_args()
+
+    if args.env_name is not None:
+        # if user passed -n command line argument, generate only that file
+        make_file(args.env_name)
+    else:
+        # otherwise, make all environment files
+        for env_name in envs:
+            make_file(env_name)

environments/requirements.txt

@@ -1,53 +1,39 @@
-# required dependencies
-geopandas
-networkx
-numpy
-pandas
-requests
-shapely
-
-# optional dependencies
-matplotlib
-rasterio
-rio-vrt
-scipy
-scikit-learn
-
-# helpful extras
+# Do not edit this file. It is automatically generated by the script
+# environments/make-env-files.py using the environment definition data in
+# environments/environments.json and the requirements in pyproject.toml.
 bottleneck
 cartopy
+conda-smithy
 folium
-jupyterlab
-numexpr
-pillow
-pysal
-python-igraph
-seaborn
-statsmodels
-
-# docs
 furo
-sphinx
-sphinx-autodoc-typehints
-
-# packaging
-conda-smithy
+geopandas>=1.0
 hatch
-pip
-twine
-validate-pyproject
-
-# typing
-mypy
-pandas-stubs
-typeguard
-types-requests
-
-# linting/testing
+jupyterlab
 lxml
+matplotlib>=3.5
 nbdime
 nbqa
+networkx>=2.5
+numexpr
+numpy>=1.22
+pandas>=1.4
+pillow
+pip
 pre-commit
+pysal>24
 pytest
 pytest-cov
-ruff
+python-igraph
+rasterio>=1.3
+requests>=2.27
+rio-vrt>=0.3
+scikit-learn>=0.23
+scipy>=1.5
+seaborn
+shapely>=2.0
+sphinx-autodoc-typehints
+sphinx>=7
+statsmodels
+twine
+typeguard
+validate-pyproject

environments/tests/env-ci.yml

@@ -1,32 +1,27 @@
+# Do not edit this file. It is automatically generated by the script
+# environments/make-env-files.py using the environment definition data in
+# environments/environments.json and the requirements in pyproject.toml.
 name: env-ci
-
 channels:
-  - conda-forge
-
+- conda-forge
 dependencies:
-  # requirements
-  - geopandas
-  - networkx
-  - numpy
-  - pandas
-  - requests
-  - shapely
-
-  # extras
-  - matplotlib
-  - rasterio
-  - rio-vrt
-  - scikit-learn
-  - scipy
-
-  # linting/testing
-  - lxml
-  - pre-commit
-  - pytest
-  - pytest-cov
-  - typeguard
-
-  # docs
-  - furo
-  - sphinx=7
-  - sphinx-autodoc-typehints
+- furo
+- geopandas>=1.0
+- lxml
+- matplotlib>=3.5
+- networkx>=2.5
+- numpy>=1.22
+- pandas>=1.4
+- pre-commit
+- pytest
+- pytest-cov
+- python>=3.9
+- rasterio>=1.3
+- requests>=2.27
+- rio-vrt>=0.3
+- scikit-learn>=0.23
+- scipy>=1.5
+- shapely>=2.0
+- sphinx-autodoc-typehints
+- sphinx>=7
+- typeguard