Docs: update ReadTheDocs configuration and build process (#4203)

The `.readthedocs.yml` configuration file is updated to ask RTD to also
install the `aiida-core` package. This is necessary for the entry points
to be properly installed, which is turn is necessary for the automated
API documentation to be successfully built. This was always failing but
went unnoticed because the nitpick option was not enabled. This is also
changed such that failures like these will fail the build from now on.

Due to the package now being installed through `pip`  it is no longer
necessary, nor preferable, to hack it into the `sys.path` in the source
configuration file. It also allows to remote the change of the previous
commit that manualy called `reentry scan`, since this was not the actual
reason for the entry points not being available.

Since the package is now installed directly through `pip` the separate
`docs/requirements_for_rtd.txt` file is now obsolete and it is removed
along with all the pre-commit hooks and scripts that automatically
updated it.

Finally, delete folder `aiida/sphinxext/tests` which became obsolete
when all tests were moved to the `tests` folder.

.gitignore

@@ -9,6 +9,7 @@
 */*/.DS_Store
 */*/*/.DS_Store
 .metadata
+.ipynb_checkpoints
 
 /venv*/
 /.idea/
@@ -28,3 +29,7 @@ coverage.xml
 # Build files
 dist/
 pip-wheel-metadata
+
+# Docs
+docs/build
+docs/source/apidoc

.pre-commit-config.yaml

@@ -133,19 +133,6 @@ repos:
                 utils/dependency_management.py
             )$
 
-    -   id: rtd-requirements
-        name: Validate docs/requirements_for_rtd.txt
-        entry: python ./utils/dependency_management.py validate-rtd-reqs
-        language: system
-        pass_filenames: false
-        files: >-
-            (?x)^(
-                setup.json|
-                setup.py|
-                utils/dependency_management.py|
-                docs/requirements_for_rtd.txt|
-            )$
-
     -   id: pyproject
         name: Validate pyproject.toml
         entry: python ./utils/dependency_management.py validate-pyproject-toml

.readthedocs.yml

@@ -1,14 +1,22 @@
-# Configuration for readthedocs
+version: 2
 
 # Important: we need to disable all unneeded formats.
-# Note that HTML and JSON are always built:
-# https://docs.readthedocs.io/en/latest/yaml-config.html#formats
-# Especially, the 'htmlzip' format takes a LOT of memory and causes
-# the build to fail - see our issue #1472:
+# Note that HTML and JSON are always built: https://docs.readthedocs.io/en/latest/yaml-config.html#formats
+# Especially, the 'htmlzip' format takes a LOT of memory and causes the build to fail - see our issue #1472:
 # https://github.com/aiidateam/aiida-core/issues/1472
 formats: []
 
-## For now I don't specify any other parameter, that is
-## currently setup in the web page of Read the Docs.
-## For other parameters see
-## https://docs.readthedocs.io/en/latest/yaml-config.html
+# Need to install the package itself such that the entry points are installed and the API doc can build properly
+python:
+    version: 3.8
+    install:
+        - method: pip
+          path: .
+          extra_requirements:
+            - docs
+            - tests
+
+# Let the build fail if there are any warnings
+sphinx:
+    builder: html
+    fail_on_warning: true

docs/source/conf.py

@@ -21,22 +21,13 @@
 import os
 import sys
 
-# Add `aiida` to the path so it can be imported without installing it
-sys.path.append(os.path.join(os.path.split(__file__)[0], os.pardir, os.pardir))
-
 import aiida
 from aiida.manage.configuration import load_documentation_profile
 
-# a reentry scan is required in RTD, to ensure all the API can be imported
-import reentry
-reentry.manager.scan()
-
 # Load the dummy profile even if we are running locally, this way the documentation will succeed even if the current
 # default profile of the AiiDA installation does not use a Django backend.
 load_documentation_profile()
 
-ON_RTD = os.environ.get('READTHEDOCS', None)
-
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.

utils/dependency_management.py

@@ -118,18 +118,6 @@ def _parse_working_set(entries):
         yield _Entry(req)
 
 
-def _generate_rtd_requirement_set():
-    """Generate content of docs/requirements_for_rtd.txt file."""
-
-    # Read the requirements from 'setup.json'
-    setup_cfg = _load_setup_cfg()
-    install_requirements = {Requirement.parse(r) for r in setup_cfg['install_requires']}
-    for key in ('tests', 'docs', 'rest', 'atomic_tools'):
-        install_requirements.update({Requirement.parse(r) for r in setup_cfg['extras_require'][key]})
-
-    return install_requirements
-
-
 @click.group()
 def cli():
     """Manage dependencies of the aiida-core package."""
@@ -172,15 +160,6 @@ def generate_environment_yml():
         )
 
 
-@cli.command('generate-rtd-reqs')
-def generate_requirements_for_rtd():
-    """Generate 'docs/requirements_for_rtd.txt' file."""
-    install_requirements = _generate_rtd_requirement_set()
-
-    with open(ROOT / Path('docs', 'requirements_for_rtd.txt'), 'w') as handle:
-        handle.write('{}\n'.format('\n'.join(sorted(map(str, install_requirements)))))
-
-
 @cli.command()
 def generate_pyproject_toml():
     """Generate 'pyproject.toml' file."""
@@ -211,7 +190,6 @@ def generate_pyproject_toml():
 def generate_all(ctx):
     """Generate all dependent requirement files."""
     ctx.invoke(generate_environment_yml)
-    ctx.invoke(generate_requirements_for_rtd)
     ctx.invoke(generate_pyproject_toml)
 
 
@@ -287,23 +265,6 @@ def validate_environment_yml():  # pylint: disable=too-many-branches
     click.secho('Conda dependency specification is consistent.', fg='green')
 
 
-@cli.command('validate-rtd-reqs', help="Validate 'docs/requirements_for_rtd.txt'.")
-def validate_requirements_for_rtd():
-    """Validate that 'docs/requirements_for_rtd.txt' is consistent with 'setup.json'."""
-
-    # Read the requirements from 'setup.json'
-    install_requirements = _generate_rtd_requirement_set()
-
-    with open(ROOT / Path('docs', 'requirements_for_rtd.txt')) as reqs_file:
-        reqs = {Requirement.parse(r) for r in reqs_file}
-
-    if reqs != install_requirements:
-        click.echo('{}'.format(reqs - install_requirements))
-        raise DependencySpecificationError("The requirements for RTD are inconsistent with 'setup.json'.")
-
-    click.secho('RTD requirements specification is consistent.', fg='green')
-
-
 @cli.command('validate-pyproject-toml', help="Validate 'pyproject.toml'.")
 def validate_pyproject_toml():
     """Validate that 'pyproject.toml' is consistent with 'setup.json'."""
@@ -347,11 +308,9 @@ def validate_all(ctx):
     - setup.json
     - environment.yml
     - pyproject.toml
-    - docs/requirements_for_rtd.txt
     """
 
     ctx.invoke(validate_environment_yml)
-    ctx.invoke(validate_requirements_for_rtd)
     ctx.invoke(validate_pyproject_toml)