Refactor assets and files into separate commands

Signed-off-by: guillemdb <guillem@fragile.tech>

mloq.yaml

@@ -13,7 +13,6 @@ license:
   license: MIT
   copyright_year: 2020
   copyright_holder: ${globals.owner}
-  open_source: ${globals.open_source}
   project_name: ${globals.project_name}
   email: ${globals.email}
   project_url: ${globals.project_url}
@@ -38,6 +37,8 @@ docs:
   author: ${globals.author}
   copyright_holder: ${license.copyright_holder}
   copyright_year: ${license.copyright_year}
+  default_branch: ${globals.default_branch}
+  deploy_docs: true
 
 git:
   disable: true

src/mloq/assets/docs/deploy-docs.yml

@@ -0,0 +1,50 @@
+name: Pages
+on:
+  push:
+    branches:
+    - {{docs.default_branch}}
+
+env:
+  NOTEBOOKS_SRC_DIR: "../notebooks"
+  NOTEBOOKS_BUILD_DIR: "./source/notebooks"
+  PIP_CACHE: |
+    ~/.cache/pip
+    ~/.local/bin
+    ~/.local/lib/python3.*/site-packages
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/setup-python@v2
+      with:
+        python-version: "3.8"
+    - uses: actions/checkout@master
+      with:
+        fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
+    - name: actions/cache
+      uses: actions/cache@v2
+      with:
+        path: ${{'{{'}} env.PIP_CACHE {{'}}'}}
+        key: ubuntu-20.04-pip-docs-${{'{{'}} hashFiles('requirements.txt') {{'}}'}}
+        restore-keys: ubuntu-20.04-pip-docs-
+    - name: Install package and dependencies
+      run: |
+        set -x
+        pip install -r requirements.txt
+        pip install .
+        if [ -e "${NOTEBOOKS_SRC_DIR}" ] && [ -e "${NOTEBOOKS_BUILD_DIR}" ]; then \
+          echo "${NOTEBOOKS_BUILD_DIR} Updating notebook folder."; \
+          rm -rf "${NOTEBOOKS_BUILD_DIR}"; \
+          cp -r "${NOTEBOOKS_SRC_DIR}"  "${NOTEBOOKS_BUILD_DIR}"; \
+        fi
+    - name: Build and Commit
+      uses: sphinx-notes/pages@v2
+      with:
+        documentation_path: ./docs/source
+        requirements_path: ./docs/requirements-docs.txt
+    - name: Push changes
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{'{{'}} secrets.GITHUB_TOKEN {{'}}'}}
+        branch: gh-pages

src/mloq/assets/templates/make_bat.txt → src/mloq/assets/docs/make_bat.txt

@@ -25,6 +25,7 @@ if errorlevel 9009 (
 	exit /b 1
 )
 
+
 %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
 goto end
 

src/mloq/assets/templates/makefile_docs.txt → src/mloq/assets/docs/makefile_docs.txt

@@ -7,6 +7,7 @@ SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source
 BUILDDIR      = build
+NOTEBOOKS_DIR = .
 
 # Put it first so that "make" without argument is like "make help".
 help:

src/mloq/assets/templates/mloq.yaml → src/mloq/assets/mloq/mloq.yaml

@@ -13,7 +13,6 @@ license:
   license: ???
   copyright_year: ${current_year:}
   copyright_holder: ${globals.owner}
-  open_source: ${globals.open_source}
   project_name: ${globals.project_name}
   project_url: ${globals.project_url}
   email: ${globals.email}

src/mloq/assets/templates/Makefile → src/mloq/assets/shared/Makefile

@@ -27,21 +27,6 @@ test:
 test-codecov:
 	find -name "*.pyc" -delete
 	pytest -n $n -s -o log_cli=true -o log_cli_level=info --cov=./ --cov-report=xml --cov-config=pyproject.toml{% endif %}
-{% if package.pipenv %}
-.PHONY: pipenv-install
-pipenv-install:
-	rm -rf *.egg-info && rm -rf build && rm -rf __pycache__
-	rm -f Pipfile && rm -f Pipfile.lock
-	pipenv install --dev -r requirements-test.txt
-	pipenv install --pre --dev -r requirements-lint.txt
-	pipenv install -r requirements.txt
-	pipenv install -e .
-	pipenv lock
-
-.PHONY: pipenv-test
-pipenv-test:
-	find -name "*.pyc" -delete
-	pipenv run pytest -s{% endif %}
 {% if docker.makefile %}
 .PHONY: docker-shell
 docker-shell:

src/mloq/assets/templates/pyproject.toml → src/mloq/assets/shared/pyproject.toml

@@ -1,3 +1,4 @@
+{% if package %}
 [tool.poetry]
 name = "{{ package.project_name }}"
 version = "0.0.0"
@@ -28,7 +29,7 @@ classifiers = [
 ]
 
 [tool.poetry.build]
-generate-setup-file = false
+generate-setup-file = true
 
 # Requirements
 [tool.poetry.dependencies]
@@ -60,7 +61,7 @@ pre-commit = "^2.15"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"
+build-backend = "poetry.core.masonry.api"{% endif %}
 
 # black is the tool to format the source code
 [tool.black]

src/mloq/commands/ci.py

@@ -7,12 +7,18 @@
 import param
 
 from mloq.command import Command
-from mloq.files import push_python_wkf
+from mloq.commands.package import DEFAULT_PYTHON_VERSIONS, PYTHON_VERSIONS
+from mloq.files import ASSETS_PATH, file
 
 
-WORKFLOW_FILES = [push_python_wkf]
+CI_ASSETS_PATH = ASSETS_PATH / "ci"
+push_python_wkf = file(
+    "push.yml",
+    CI_ASSETS_PATH,
+    "GitHub Actions continuous integration workflow file",
+)
 
-PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"]
+WORKFLOW_FILES = [push_python_wkf]
 
 
 class CiCMD(Command):
@@ -29,7 +35,7 @@ class CiCMD(Command):
     bot_email = param.String(doc="Bot account email")
     ci_python_version = param.String(doc="Primary Python version in GitHub Actions")
     python_versions = param.ListSelector(
-        default=PYTHON_VERSIONS,
+        default=DEFAULT_PYTHON_VERSIONS,
         doc="Supported python versions",
         objects=PYTHON_VERSIONS,
     )

src/mloq/commands/docker.py

@@ -12,14 +12,16 @@
     tensorflow_req,
 )
 from mloq.config.param_patch import param
-from mloq.files import DOCKER_PATH, file
+from mloq.files import ASSETS_PATH, file
 
 
-dockerfile = file("Dockerfile", DOCKER_PATH, description="Docker container for the project")
+DOCKER_ASSETS_PATH = ASSETS_PATH / "docker"
+dockerfile = file("Dockerfile", DOCKER_ASSETS_PATH, description="Docker container for the project")
 makefile_docker = file(
     "Makefile.docker",
-    DOCKER_PATH,
+    DOCKER_ASSETS_PATH,
     description="Makefile for the Docker container setup",
+    is_static=True,
 )
 DOCKER_FILES = [dockerfile, makefile_docker]
 

src/mloq/commands/docs.py

@@ -7,10 +7,49 @@
 import param
 
 from mloq.command import Command
-from mloq.files import conf_py, docs_req, index_md, make_bat_docs, makefile_docs
+from mloq.files import ASSETS_PATH, file
 
 
-DOCS_FILES = [conf_py, index_md, makefile_docs, make_bat_docs, docs_req]
+DOCS_ASSETS_PATH = ASSETS_PATH / "docs"
+# Documentation files
+makefile_docs = file(
+    "makefile_docs.txt",
+    DOCS_ASSETS_PATH,
+    "common make commands for building the documentation",
+    dst="Makefile",
+    is_static=True,
+)
+make_bat_docs = file(
+    "make_bat.txt",
+    DOCS_ASSETS_PATH,
+    "common make commands for building the documentation",
+    dst="make.bat",
+    is_static=True,
+)
+docs_req = file(
+    "requirements-docs.txt",
+    DOCS_ASSETS_PATH,
+    "list of exact versions of the packages needed to build your documentation",
+    is_static=True,
+)
+conf_py = file(
+    "conf.txt",
+    DOCS_ASSETS_PATH,
+    "configuration file for sphinx and doc plugins",
+    dst="conf.py",
+)
+index_md = file(
+    "index.md",
+    DOCS_ASSETS_PATH,
+    "configuration file for sphinx and doc plugins",
+)
+deploy_docs = file(
+    "deploy-docs.yml",
+    DOCS_ASSETS_PATH,
+    "workflow for deploying the documentation to GitHub pages",
+)
+
+DOCS_FILES = [conf_py, index_md, makefile_docs, make_bat_docs, docs_req, deploy_docs]
 
 
 class DocsCMD(Command):
@@ -23,12 +62,19 @@ class DocsCMD(Command):
     author = param.String("${globals.author}", doc="Author(s) of the project")
     copyright_year = param.Integer(doc="Year when the project started")
     copyright_holder = param.String("${docs.author}", doc="Copyright holder")
+    deploy_docs = param.Boolean(True, doc="Deploy docs to GitHub Pages?")
+    default_branch = param.String("${globals.default_branch}", doc="Branch used to build the docs")
     files = tuple(DOCS_FILES)
 
     @property
     def directories(self) -> Tuple[Path]:
         """Tuple containing Paths objects representing the directories created by the command."""
-        return tuple([Path("docs") / "source" / "markdown"])
+        docs_folder = Path("docs") / "source" / "markdown"
+        paths = [docs_folder]
+        if self.deploy_docs:
+            workflow_path = Path(".github") / "workflows"
+            paths.append(workflow_path)
+        return tuple(paths)
 
     def interactive_config(self) -> DictConfig:
         """Generate the configuration of the project interactively."""
@@ -38,7 +84,16 @@ def interactive_config(self) -> DictConfig:
     def record_files(self) -> None:
         """Register the files that will be generated by mloq."""
         source_files = {"conf.py", "index.md"}
-        docs_path = Path("docs")
-        for file in self.files:
-            path = (docs_path / "source") if str(file.dst) in source_files else docs_path
-            self.record.register_file(file=file, path=path)
+        docs_ASSETS_path = Path("docs")
+        for _file in self.files:
+            if _file == deploy_docs:
+                if not self.deploy_docs:
+                    continue
+                path = Path(".github") / "workflows"
+            else:
+                path = (
+                    (docs_ASSETS_path / "source")
+                    if str(_file.dst) in source_files
+                    else docs_ASSETS_path
+                )
+            self.record.register_file(file=_file, path=path)

src/mloq/commands/license.py

@@ -2,24 +2,46 @@
 from pathlib import Path
 
 import click
-from omegaconf import DictConfig, MISSING
+from omegaconf import DictConfig
 
 from mloq.command import Command
 from mloq.config.param_patch import param
-from mloq.files import code_of_conduct, contributing, dco, LICENSES
-
-
-OPEN_SOURCE_FILES = [dco, contributing, code_of_conduct]
+from mloq.files import ASSETS_PATH, file
+
+
+TEMPLATES_PATH = ASSETS_PATH / "license"
+dco = file(
+    "DCO.md",
+    TEMPLATES_PATH,
+    "Developer Certificate of Origin - needed in open source projects to certify that "
+    "the incoming contributions are legitimate",
+    is_static=True,
+)
+mit_license = file("MIT_LICENSE", TEMPLATES_PATH, "license of the project", dst="LICENSE")
+apache_license = file("APACHE_LICENSE", TEMPLATES_PATH, "license of the project", dst="LICENSE")
+gpl_license = file(
+    "GPL_LICENSE",
+    TEMPLATES_PATH,
+    "license of the project",
+    dst="LICENSE",
+    is_static=True,
+)
+LICENSES = {
+    "MIT": mit_license,
+    "Apache-2.0": apache_license,
+    "GPL-3.0": gpl_license,
+}
+
+LICENSE_FILES = [file for file in LICENSES.values()] + [dco]
 
 
 class LicenseCMD(Command):
     """Implement the functionality of the license Command."""
 
     cmd_name = "license"
-    files = tuple([file for file in LICENSES.values()] + OPEN_SOURCE_FILES)
+    files = tuple(LICENSE_FILES)
     LICENSES = LICENSES
     disable = param.Boolean(default=None, doc="Disable license command?")
-    open_source = param.Boolean(MISSING, doc="Is the project Open Source?")
     license = param.String("MIT", doc="Project license type")
     copyright_year = param.Integer("${current_year:}", doc="Year when the project started")
     copyright_holder = param.String("${globals.owner}", doc="Copyright holder")
@@ -35,8 +57,7 @@ def interactive_config(self) -> DictConfig:
     def record_files(self) -> None:
         """Register the files that will be generated by mloq."""
         conf = self.record.config.license
-        if conf.open_source:
-            for file in OPEN_SOURCE_FILES:
-                self.record.register_file(file=file, path=Path())
+        self.record.register_file(file=dco, path=Path())
+        if conf.license != "proprietary":
             license_file = self.LICENSES[conf.license]
             self.record.register_file(file=license_file, path=Path())

src/mloq/commands/lint.py

@@ -6,17 +6,23 @@
 
 from mloq.command import Command
 from mloq.config.param_patch import param
-from mloq.files import lint_req, pyproject_toml
+from mloq.files import ASSETS_PATH, file, pyproject_toml
 
 
-PACKAGE_FILES = [pyproject_toml, lint_req]
+lint_req = file(
+    "requirements-lint.txt",
+    ASSETS_PATH / "lint",
+    "list of exact versions of the packages used to check your code style",
+    is_static=True,
+)
+LINT_FILES = [pyproject_toml, lint_req]
 
 
 class LintCMD(Command):
     """Implement the functionality of the lint Command."""
 
     cmd_name = "lint"
-    files = tuple(PACKAGE_FILES)
+    files = tuple(LINT_FILES)
     disable = param.Boolean(default=None, doc="Disable lint command?")
     docstring_checks = param.Boolean(True, doc="Apply docstring checks?")
     pyproject_extra = param.String("", doc="Additional pyproject.toml configuration")
@@ -30,5 +36,5 @@ def interactive_config(self) -> DictConfig:
 
     def record_files(self) -> None:
         """Register the files that will be generated by mloq."""
-        for file in self.files:
-            self.record.register_file(file=file, path=Path())
+        for _file in self.files:
+            self.record.register_file(file=_file, path=Path())