Transition to Mkdocs for versioned api-docs

.github/workflows/publish_api_docs.yml

@@ -24,6 +24,10 @@ jobs:
     - run: npm install
     - run: npm install html-minifier -g
 
+    - name: Get the version from the github tag ref
+      id: get_version
+      run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+
     - name: Install poetry
       run: |
         python -m pip install --upgrade pip
@@ -37,12 +41,4 @@ jobs:
         python -m poetry run pip install awscli
 
     - name: Runs docs generation
-      run: poetry run bash scripts/generate-docs.sh
-
-    - name: Runs deploy
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_BUCKET: apidocs.zenml.io
-        CLOUDFRONT_DISTRIBUTION_ID: EB0YZNLQR50S5
-      run: poetry run bash scripts/deploy-docs.sh
+      run: poetry run bash scripts/generate-docs.sh -s src/zenml/ -v ${{ steps.get_version.outputs.VERSION }} --push --latest

.gitignore

@@ -170,6 +170,7 @@ local_test/
 .zen/
 !cloudbuild.yaml
 docs/book/_build/
+docs/mkdocs/api_docs(/
 zenml_examples/
 
 # GitHub Folder YAML files not to be ignored

docs/mkdocs.yml

@@ -0,0 +1,30 @@
+site_name: ZenML API Reference
+site_url: https://apidocs.zenml.io/
+
+docs_dir: mkdocs
+
+theme:
+  name: material
+  locale: en
+  favicon: _assets/favicon.png
+
+repo_url: https://github.com/zenml-io/zenml
+edit_uri: https://github.com/zenml-io/zenml/docs/
+
+plugins:
+    - search
+    - awesome-pages
+    - mkdocstrings:
+        default_handler: python
+        handlers:
+          python:
+            rendering:
+              show_source: true
+        watch:
+          - ../src/zenml
+
+copyright: "Copyright © 2022 ZenML GmbH"
+
+extra:
+  version:
+    provider: mike

docs/mkdocs/.pages

@@ -0,0 +1,4 @@
+nav:
+    - ZenML: index.md
+    - CLI docs: cli.md
+    - ...

docs/mkdocs/cli.md

@@ -0,0 +1,10 @@
+# Cli
+
+::: zenml.cli
+    handler: python
+    selection:
+      members: 
+        - "__init__.py"  
+    rendering:
+      show_root_heading: true
+      show_source: true

docs/mkdocstrings_helper.py

@@ -0,0 +1,205 @@
+from typing import List, Optional
+import os
+import subprocess
+from pathlib import Path
+import argparse
+
+PYDOCSTYLE_CMD = "pydocstyle --convention=google --add-ignore=D100,D101,D102," \
+                 "D103,D104,D105,D107,D202"
+
+API_DOCS_TITLE = '# Welcome to the ZenML Api Docs\n'
+
+API_DOCS = 'api_docs'
+
+
+def to_md_file(
+        markdown_str: str,
+        filename: str,
+        out_path: Path = Path("."),
+) -> None:
+    """Creates an API docs file from a provided text.
+
+    Args:
+        markdown_str (str): Markdown string with line breaks to write to file.
+        filename (str): Filename without the .md
+        out_path (str): The output directory
+    """
+    if not markdown_str:
+        # Don't write empty files
+        return
+
+    md_file = filename
+    if not filename.endswith(".md"):
+        md_file = filename + ".md"
+
+    print(f"Writing {md_file}.")
+    with open(os.path.join(out_path, md_file), "w", encoding="utf-8") as f:
+        f.write(markdown_str)
+
+
+def _is_module_ignored(module_name: str, ignored_modules: List[str]) -> bool:
+    """Checks if a given module is ignored."""
+    if module_name.split(".")[-1].startswith("_"):
+        return True
+
+    for ignored_module in ignored_modules:
+        if module_name == ignored_module:
+            return True
+
+        # Check is module is subpackage of an ignored package
+        if module_name.startswith(ignored_module + "."):
+            return True
+
+    return False
+
+
+def generate_title(s: str) -> str:
+    """Remove underscores and capitalize first letter to each word"""
+    s = s.replace('_', ' ')
+    s = s.title()
+    return s
+
+
+def create_entity_docs(api_doc_file_dir: Path,
+                       ignored_modules: List[str],
+                       sources_path: Path,
+                       index_file_contents: Optional[List[str]]
+                       ) -> Optional[List[str]]:
+    for item in sources_path.iterdir():
+        if item.name not in ignored_modules:
+            item_name = generate_title(item.stem)
+
+            # Extract zenml import path from sources path
+            # Example: src/zenml/cli/function.py ->  zenml.cli
+            zenml_import_path = '.'.join(item.parts[1:-1])
+
+            module_md = f"# {item_name}\n\n" \
+                        f"::: {zenml_import_path}.{item.stem}\n" \
+                        f"    handler: python\n" \
+                        f"    rendering:\n" \
+                        f"      show_root_heading: true\n" \
+                        f"      show_source: true\n"
+
+            to_md_file(
+                module_md,
+                item.stem,
+                out_path=api_doc_file_dir,
+            )
+
+            if index_file_contents:
+                index_entry = f"# [{item_name}]" \
+                              f"({api_doc_file_dir.name}/{item.stem})\n\n" \
+                              f"::: zenml.{item.stem}\n" \
+                              f"    handler: python\n" \
+                              f"    selection:\n" \
+                              f"        members: false\n"
+
+                index_file_contents.append(index_entry)
+
+    return index_file_contents
+
+
+def create_cli_docs(cli_dev_doc_file_dir: Path,
+                    ignored_modules: List[str],
+                    sources_path: Path,
+                    ) -> None:
+    # TODO [MED]: Find Solution for issue with click-decorated functions
+    #  Some resources concerning this issue can be found here
+    #  https://github.com/mkdocstrings/mkdocstrings/issues/162
+    #  https://mkdocstrings.github.io/troubleshooting/#my-wrapped-function-shows-documentationcode-for-its-wrapper-instead-of-its-own
+    create_entity_docs(
+        api_doc_file_dir=cli_dev_doc_file_dir, ignored_modules=ignored_modules,
+        sources_path=sources_path, index_file_contents=None
+    )
+
+
+def generate_docs(
+        path: Path,
+        output_path: Path,
+        ignored_modules: Optional[List[str]] = None,
+        validate: bool = False
+) -> None:
+    """Generates top level separation of primary entities inside th zenml source
+    directory for the purpose of generating mkdocstring markdown files.
+
+    Args:
+        path: Selected paths or import name for markdown generation.
+        output_path: The output path for the creation of the markdown files.
+        ignored_modules: A list of modules that should be ignored.
+        validate: Boolean if pydocstyle should be verified within dir
+    """
+    # Set up output paths for the generated md files
+    api_doc_file_dir = output_path / API_DOCS
+    cli_dev_doc_file_dir = output_path / API_DOCS / 'cli'
+
+    api_doc_file_dir.mkdir(parents=True, exist_ok=True)
+    cli_dev_doc_file_dir.mkdir(parents=True, exist_ok=True)
+
+    if not ignored_modules:
+        ignored_modules = list()
+
+    # The Cli docs are treated differently as the user facing docs need to be
+    # split from the developer-facing docs
+    ignored_modules.append('cli')
+
+    # Validate that all docstrings conform to pydocstyle rules
+    if (validate and Path(path).is_dir() and
+            subprocess.call(f"{PYDOCSTYLE_CMD} {path}", shell=True) > 0):
+        raise Exception(f"Validation for {path} failed.")
+
+    index_file_contents = [API_DOCS_TITLE]
+
+    index_file_contents = create_entity_docs(
+        api_doc_file_dir=api_doc_file_dir, ignored_modules=ignored_modules,
+        sources_path=path, index_file_contents=index_file_contents
+    )
+
+    create_cli_docs(
+        cli_dev_doc_file_dir=cli_dev_doc_file_dir,
+        ignored_modules=ignored_modules,
+        sources_path=path / 'cli',
+    )
+
+    index_file_str = '\n'.join(index_file_contents)
+    to_md_file(
+        index_file_str,
+        'index.md',
+        out_path=output_path,
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Arguments to run the'
+                                                 'mkdocstrings helper')
+    # Required positional argument
+    parser.add_argument('--path',
+                        type=str,
+                        default='src/zenml',
+                        help='Location of the source code')
+
+    # Optional positional argument
+    parser.add_argument('--output_path',
+                        type=str,
+                        default='docs/mkdocs',
+                        help='Location at which to generate the markdown file')
+
+    # Optional argument
+    parser.add_argument('--ignored_modules',
+                        type=List[str],
+                        default=['VERSION', 'README.md',
+                                 '__init__.py', '__pycache__'],
+                        help='Top level entities that should not end up in '
+                             'the api docs (e.g. README.md, __init__')
+
+    # Switch
+    parser.add_argument('--validate',
+                        action='store_true',
+                        help='A boolean switch')
+
+    args = parser.parse_args()
+    print(args.validate)
+
+    generate_docs(path=Path(args.path),
+                  output_path=Path(args.output_path),
+                  ignored_modules=args.ignored_modules,
+                  validate=args.validate)

pyproject.toml

@@ -82,6 +82,15 @@ sphinx-copybutton = "^0.4.0"
 typing-extensions = ">=3.7.4"
 pytest-randomly = "^3.10.1"
 
+# mkdocs including plugins
+mkdocs="^1.2.3"
+mkdocs-material="^8.1.7"
+mkdocs-awesome-pages-plugin="^2.6.1"
+mkdocstrings="^0.17.0"
+pydocstyle="^6.1.1"
+mike="^1.1.2"
+
+
 # mypy type stubs
 types-certifi = "^2021.10.8.0"
 types-croniter = "^1.0.2"

scripts/generate-docs.sh

@@ -1,16 +1,58 @@
 #!/bin/sh -e
 set -x
 
-SRC=${1:-"src/zenml/"}
+SRC="src/zenml/"
+PUSH=""
+LATEST=""
+
+msg() {
+  echo >&2 -e "${1-}"
+}
+
+die() {
+  msg=$1
+  code=${2-1} # default exit status 1
+  msg "$msg"
+  exit "$code"
+}
+
+while :; do
+  case "${1-}" in
+  -p | --push) PUSH="true";;
+  -l | --latest) LATEST="latest";;
+  -s | --source)
+    SRC="${2-}"
+    shift
+    ;;
+  -v | --version)
+    VERSION="${2-}"
+    shift
+    ;;
+  -?*) die "Unknown option: $1" ;;
+  *) break ;;
+  esac
+  shift
+done
+
+# check required params and arguments
+[ -z "${VERSION-}" ] && die "Missing required parameter: VERSION. Please supply a doc version"
 
 export ZENML_DEBUG=1
 export ZENML_ANALYTICS_OPT_IN=false
-rm -rf docs/sphinx_docs/_build/ || true
-rm -rf docs/sphinx_docs/api_reference || true
+rm -rf docs/mkdocs/api_docs || true
 
-# only show documented members of a module
-export SPHINX_APIDOC_OPTIONS=members,ignore-module-all
+python docs/mkdocstrings_helper.py --path $SRC --output_path docs/mkdocs/
 
-sphinx-apidoc --force --separate --no-toc --module-first --templatedir docs/sphinx_docs/_templates -o docs/sphinx_docs/api_reference $SRC
-cd docs/sphinx_docs/
-make html
+if [ -n "$PUSH" ]; then
+  if [ -n "$LATEST" ]; then
+    mike deploy --push --update-aliases --config-file docs/mkdocs.yml $VERSION latest
+  else
+    mike deploy --push --update-aliases --config-file docs/mkdocs.yml $VERSION
+  fi
+else
+  if [ -n "$LATEST" ]; then
+    mike deploy --update-aliases --config-file docs/mkdocs.yml $VERSION latest
+  else
+    mike deploy --update-aliases --config-file docs/mkdocs.yml $VERSION
+  fi
+fi