diff --git a/docs/.vuepress/config.js b/docs/.vuepress/config.js index 7a1a57d968..dc0391af16 100644 --- a/docs/.vuepress/config.js +++ b/docs/.vuepress/config.js @@ -33,6 +33,10 @@ module.exports = { text: 'Guide', link: '/guide/' }, + { + text: 'Notebooks', + link: '/notebooks/' + }, { text: 'Reference', link: '/reference/' diff --git a/docs/autodoc.py b/docs/autodoc.py index 318a80cdf0..529e104b9d 100644 --- a/docs/autodoc.py +++ b/docs/autodoc.py @@ -6,77 +6,22 @@ import importlib import inspect import json +import logging import os import pkgutil import re import sys +import urllib from functools import lru_cache from glob import glob +from typing import List, Tuple import skdecide -refs = set() -header_comment = "# %%\n" - - -# https://github.com/kiwi0fruit/ipynb-py-convert/blob/master/ipynb_py_convert/__main__.py -def py2nb(py_str, title=None): - cells = [] - if title is not None: - # first cell = title - cell = { - "cell_type": "markdown", - "metadata": {}, - "source": [f"# {title}"], - } - cells.append(cell) - - chunks = py_str.split(f"\n\n{header_comment}")[1:] - for chunk in chunks: - cell_type = "code" - chunk = chunk.strip() - if chunk.startswith('"""'): - chunk = chunk.strip('"\n') - cell_type = "markdown" - elif chunk.startswith("# %"): - # magic commands - chunk = chunk[2:] - - cell = { - "cell_type": cell_type, - "metadata": {}, - "source": chunk.splitlines(True), - } - - if cell_type == "code": - cell.update({"outputs": [], "execution_count": None}) - - cells.append(cell) - - notebook = { - "cells": cells, - "metadata": { - "anaconda-cloud": {}, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3", - }, - "language_info": { - "codemirror_mode": {"name": "ipython", "version": 3}, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1", - }, - }, - "nbformat": 4, - "nbformat_minor": 1, - } +NOTEBOOKS_LIST_PLACEHOLDER = "[[notebooks-list]]" - return notebook +logger = logging.getLogger(__name__) +refs = set() # https://stackoverflow.com/questions/48879353/how-do-you-recursively-get-all-submodules-in-a-python-package @@ -227,6 +172,110 @@ def is_implemented(func_code): return not func_code.strip().endswith("raise NotImplementedError") +def get_binder_link( + binder_env_repo_name: str, + binder_env_branch: str, + notebooks_repo_url: str, + notebooks_branch: str, + notebook_relative_path: str, +) -> str: + # binder hub url + jupyterhub = urllib.parse.urlsplit("https://mybinder.org") + + # path to the binder env + binder_path = f"v2/gh/{binder_env_repo_name}/{binder_env_branch}" + + # nbgitpuller query + notebooks_repo_basename = os.path.basename(notebooks_repo_url) + urlpath = f"tree/{notebooks_repo_basename}/{notebook_relative_path}" + next_url_params = urllib.parse.urlencode( + { + "repo": notebooks_repo_url, + "urlpath": urlpath, + "branch": notebooks_branch, + } + ) + next_url = f"git-pull?{next_url_params}" + query = urllib.parse.urlencode({"urlpath": next_url}) + + # full link + link = urllib.parse.urlunsplit( + urllib.parse.SplitResult( + scheme=jupyterhub.scheme, + netloc=jupyterhub.netloc, + path=binder_path, + query=query, + fragment="", + ) + ) + + return link + + +def get_github_link( + notebooks_repo_url: str, + notebooks_branch: str, + notebook_relative_path: str, +) -> str: + return f"{notebooks_repo_url}/blob/{notebooks_branch}/{notebook_relative_path}" + + +def get_repo_n_branches_for_binder_n_github_links() -> Tuple[bool, str, str, str, str]: + # repos + branches to use for binder environment and notebooks content. + creating_links = True + try: + binder_env_repo_name = os.environ["AUTODOC_BINDER_ENV_GH_REPO_NAME"] + except KeyError: + binder_env_repo_name = "airbus/scikit-decide" + try: + binder_env_branch = os.environ["AUTODOC_BINDER_ENV_GH_BRANCH"] + except KeyError: + binder_env_branch = "binder" + try: + notebooks_repo_url = os.environ["AUTODOC_NOTEBOOKS_REPO_URL"] + notebooks_branch = os.environ["AUTODOC_NOTEBOOKS_BRANCH"] + except KeyError: + # missing environment variables => no github and binder links creation + notebooks_repo_url = "" + notebooks_branch = "" + creating_links = False + logger.warning( + "Missing environment variables AUTODOC_NOTEBOOKS_REPO_URL " + "or AUTODOC_NOTEBOOKS_BRANCH to create github and binder links for notebooks." + ) + return ( + creating_links, + notebooks_repo_url, + notebooks_branch, + binder_env_repo_name, + binder_env_branch, + ) + + +def extract_notebook_title_n_description( + notebook_filepath: str, +) -> Tuple[str, List[str]]: + # load notebook + with open(notebook_filepath, "rt") as f: + notebook = json.load(f) + + # find title + description: from first cell, h1 title + remaining text. + # or title from filename else + title = "" + description_lines: List[str] = [] + cell = notebook["cells"][0] + if cell["cell_type"] == "markdown": + if cell["source"][0].startswith("# "): + title = cell["source"][0][2:].strip() + description_lines = cell["source"][1:] + else: + description_lines = cell["source"] + if not title: + title = os.path.splitext(os.path.basename(notebook_filepath))[0] + + return title, description_lines + + if __name__ == "__main__": docdir = os.path.dirname(os.path.abspath(__file__)) @@ -570,35 +619,57 @@ def is_implemented(func_code): with open(f"{docdir}/.vuepress/_state.json", "w") as f: json.dump(state, f) - # Convert selected examples to notebooks & write Examples page (guide/_examples.md) - examples = "# Examples\n\n" - - selected_examples = [] - for example in glob(f"{docdir}/../examples/*.py"): - docstr, name, code = py_parse(example) - if docstr.startswith("Example "): - selected_examples.append((docstr, name, code)) - - os.makedirs(f"{docdir}/.vuepress/public/notebooks", exist_ok=True) - sorted_examples = sorted(selected_examples) - for docstr, name, code in sorted_examples: - title = docstr[docstr.index(":") + 1 :] - examples += f"## {title}\n\n" - examples += f'Download Notebook\n' - examples += f'Run in Google Colab\n\n' - notebook = py2nb(code, title=title) - - # Render cells, except for first cell with title - for cell in notebook["cells"][1:]: - cell_type = cell["cell_type"] - cell_source = "".join(cell["source"]) - if cell_type == "markdown": - examples += f"{cell_source}\n\n" - elif cell_type == "code": - examples += f"``` py\n{cell_source}\n```\n\n" - - with open(f"{docdir}/.vuepress/public/notebooks/{name}.ipynb", "w") as f: - json.dump(notebook, f, indent=2) - - with open(f"{docdir}/guide/_examples.md", "w") as f: - f.write(examples) + # List existing notebooks and and write Notebooks page + rootdir = os.path.abspath(f"{docdir}/..") + notebook_filepaths = sorted(glob(f"{rootdir}/notebooks/*.ipynb")) + notebooks_list_text = "" + ( + creating_links, + notebooks_repo_url, + notebooks_branch, + binder_env_repo_name, + binder_env_branch, + ) = get_repo_n_branches_for_binder_n_github_links() + # loop on notebooks sorted alphabetically by filenames + for notebook_filepath in notebook_filepaths: + title, description_lines = extract_notebook_title_n_description( + notebook_filepath + ) + # subsection title + notebooks_list_text += f"## {title}\n\n" + # links + if creating_links: + notebook_path_prefix_len = len(f"{rootdir}/") + notebook_relative_path = notebook_filepath[notebook_path_prefix_len:] + binder_link = get_binder_link( + binder_env_repo_name=binder_env_repo_name, + binder_env_branch=binder_env_branch, + notebooks_repo_url=notebooks_repo_url, + notebooks_branch=notebooks_branch, + notebook_relative_path=notebook_relative_path, + ) + binder_badge = ( + f"[![Binder](https://mybinder.org/badge_logo.svg)]({binder_link})" + ) + github_link = get_github_link( + notebooks_repo_url=notebooks_repo_url, + notebooks_branch=notebooks_branch, + notebook_relative_path=notebook_relative_path, + ) + github_badge = f"[![Github](https://img.shields.io/badge/see-Github-579aca?logo=github)]({github_link})" + + # markdown item + notebooks_list_text += f"{github_badge}\n{binder_badge}\n\n" + # description + notebooks_list_text += "".join(description_lines) + notebooks_list_text += "\n\n" + + with open(f"{docdir}/notebooks/README.md.template", "rt") as f: + readme_template_text = f.read() + + readme_text = readme_template_text.replace( + NOTEBOOKS_LIST_PLACEHOLDER, notebooks_list_text + ) + + with open(f"{docdir}/notebooks/README.md", "wt") as f: + f.write(readme_text) diff --git a/docs/guide/README.md b/docs/guide/README.md index aa40205cdb..bd7b5ba1d2 100644 --- a/docs/guide/README.md +++ b/docs/guide/README.md @@ -152,13 +152,13 @@ In the example of the Maze solved with Lazy A*, the goal (in green) should be re ## Examples -**Go to Examples for a curated list of Python notebooks (recommended to start).** +### Notebooks -More examples can be found in the `/examples` folder, showing how to import or define a domain, and how to run or solve it. Most of the examples rely on scikit-decide Hub, an extensible catalog of domains/solvers. +Go to the dedicated Notebooks page to see a curated list of notebooks recommended to start with scikit-decide. -**Warning**: the examples whose filename starts with an underscore are currently being migrated to the new API and might not be working in the meantime (same goes for domains/solvers inside `skdecide/hub`). +### Python scripts -**Warning**: some content currently in the hub (especially the MasterMind domain and the POMCP/CGP solvers) will require permission from their original authors before entering the public hub when open sourced. +More examples can be found in the `examples/` folder, showing how to import or define a domain, and how to run or solve it. Most of the examples rely on scikit-decide Hub, an extensible catalog of domains/solvers. ### Playground diff --git a/docs/notebooks/README.md.template b/docs/notebooks/README.md.template new file mode 100644 index 0000000000..0222b779f1 --- /dev/null +++ b/docs/notebooks/README.md.template @@ -0,0 +1,7 @@ +# Notebooks + +We present here a curated list of notebooks recommended to start with scikit-decide, available in the `notebooks/` folder of the repository. + +[[toc]] + +[[notebooks-list]]