Do not modify MANIFEST.in on install

.github/actions/pkg-check/action.yml

@@ -33,9 +33,9 @@ runs:
 
   - name: Check package
     run: |
+      git diff --exit-code || exit $?  # make sure there are no local unstaged changes
       ls -l dist/
       twine check dist/*
-      # python setup.py clean
     shell: bash
 
   - name: Unzip packages

.github/workflows/docs-checks.yml

@@ -75,7 +75,6 @@ jobs:
         if: ${{ matrix.pkg-name == 'app' }}
         run: |
           pip install -e . -U  -f https://download.pytorch.org/whl/cpu/torch_stable.html -f pypi
-          git checkout -- MANIFEST.in
 
       - name: Adjust docs refs
         if: ${{ matrix.pkg-name == 'app' }}

setup.py

@@ -39,11 +39,15 @@
     b) with a parameterization build desired packages in to standard `dist/` folder
     c) validate packages and publish to PyPI
 """
+import contextlib
 import os
+import tempfile
 from importlib.util import module_from_spec, spec_from_file_location
 from types import ModuleType
+from typing import Generator, Optional
 
-from setuptools import setup
+import setuptools
+import setuptools.command.egg_info
 
 _PACKAGE_NAME = os.environ.get("PACKAGE_NAME")
 _PACKAGE_MAPPING = {
@@ -69,6 +73,45 @@ def _load_py_module(name: str, location: str) -> ModuleType:
     return py
 
 
+def _named_temporary_file(directory: Optional[str] = None) -> str:
+    # `tempfile.NamedTemporaryFile` has issues in Windows
+    # https://github.com/deepchem/deepchem/issues/707#issuecomment-556002823
+    if directory is None:
+        directory = tempfile.gettempdir()
+    return os.path.join(directory, os.urandom(24).hex())
+
+
+@contextlib.contextmanager
+def _set_manifest_path(manifest_dir: str, aggregate: bool = False) -> Generator:
+    if aggregate:
+        # aggregate all MANIFEST.in contents into a single temporary file
+        manifest_path = _named_temporary_file(manifest_dir)
+        mapping = _PACKAGE_MAPPING.copy()
+        del mapping["lightning"]
+        lines = []
+        for pkg in mapping.values():
+            with open(os.path.join(_PATH_SRC, pkg, "MANIFEST.in")) as fh:
+                lines.extend(fh.readlines())
+        # convert lightning_foo to lightning/foo
+        for new, old in mapping.items():
+            lines = [line.replace(old, f"lightning/{new}") for line in lines]
+        fp = open(manifest_path, mode="w")
+        fp.writelines(lines)
+        fp.flush()
+    else:
+        manifest_path = os.path.join(manifest_dir, "MANIFEST.in")
+        assert os.path.exists(manifest_path)
+    # avoid error: setup script specifies an absolute path
+    manifest_path = os.path.relpath(manifest_path, _PATH_ROOT)
+    setuptools.command.egg_info.manifest_maker.template = manifest_path
+    yield
+    # cleanup
+    setuptools.command.egg_info.manifest_maker.template = "MANIFEST.in"
+    if aggregate:
+        fp.close()
+        os.remove(manifest_path)
+
+
 if __name__ == "__main__":
     setup_tools = _load_py_module(name="setup_tools", location=os.path.join(".actions", "setup_tools.py"))
 
@@ -86,13 +129,19 @@ def _load_py_module(name: str, location: str) -> ModuleType:
     # should have included only the relevant files of the package to install
     possible_packages = _PACKAGE_MAPPING.values() if _PACKAGE_NAME is None else [_PACKAGE_MAPPING[_PACKAGE_NAME]]
     for pkg in possible_packages:
-        pkg_setup = os.path.join(_PATH_SRC, pkg, "__setup__.py")
+        pkg_path = os.path.join(_PATH_SRC, pkg)
+        pkg_setup = os.path.join(pkg_path, "__setup__.py")
         if os.path.exists(pkg_setup):
             print(f"{pkg_setup} exists. Running `setuptools.setup`")
             setup_module = _load_py_module(name=f"{pkg}_setup", location=pkg_setup)
-            setup_module._adjust_manifest(pkg_name=pkg)
-            setup_args = setup_module._setup_args(pkg_name=pkg)
-            setup(**setup_args)
+            setup_args = setup_module._setup_args()
+            if _PACKAGE_NAME is None:
+                # we are installing a wheel, no need for MANIFEST.in things
+                setuptools.setup(**setup_args)
+            else:
+                # we are installing from source, set the correct manifest path
+                with _set_manifest_path(pkg_path, aggregate=pkg == "lightning"):
+                    setuptools.setup(**setup_args)
             break
     else:
         raise RuntimeError(f"Something's wrong, no package was installed. Package name: {_PACKAGE_NAME}")

src/lightning/__setup__.py

@@ -24,27 +24,7 @@ def _load_py_module(name: str, location: str) -> ModuleType:
 _SETUP_TOOLS = _load_py_module("setup_tools", os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py"))
 
 
-def _adjust_manifest(**kwargs: Any) -> None:
-    # todo: consider rather aggregation of particular manifest adjustments
-    manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in")
-    assert os.path.isfile(manifest_path)
-    with open(manifest_path) as fp:
-        lines = [ln.rstrip() for ln in fp.readlines()]
-    lines += [
-        "recursive-include src/lightning *.md",
-        "recursive-include requirements *.txt",
-        "recursive-include src/lightning/app/ui *",
-        "recursive-include src/lightning/cli/*-template *",  # Add templates as build-in
-        # fixme: this is strange, this shall work with setup find package - include
-        "prune src/lightning_app",
-        "prune src/lightning_lite",
-        "prune src/pytorch_lightning",
-    ]
-    with open(manifest_path, "w") as fp:
-        fp.writelines([ln + os.linesep for ln in lines])
-
-
-def _setup_args(**kwargs: Any) -> Dict[str, Any]:
+def _setup_args() -> Dict[str, Any]:
     _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))
     _version = _load_py_module("version", os.path.join(_PACKAGE_ROOT, "__version__.py"))
     _long_description = _SETUP_TOOLS.load_readme_description(

src/lightning_app/MANIFEST.in

@@ -0,0 +1,7 @@
+include src/lightning_app/CHANGELOG.md
+include src/lightning_app/README.md
+recursive-include requirements/app *.txt
+include .actions/setup_tools.py
+recursive-include src/lightning_app/cli/*-template *
+# TODO: remove this once lightning-ui package is ready as a dependency
+recursive-include src/lightning_app/ui *

src/lightning_app/__setup__.py

@@ -41,27 +41,7 @@ def _prepare_extras() -> Dict[str, Any]:
     return extras
 
 
-def _adjust_manifest(**__: Any) -> None:
-    manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in")
-    assert os.path.isfile(manifest_path)
-    with open(manifest_path) as fp:
-        lines = fp.readlines()
-    lines += [
-        "recursive-exclude src *.md" + os.linesep,
-        "recursive-exclude requirements *.txt" + os.linesep,
-        "recursive-include src/lightning_app *.md" + os.linesep,
-        "recursive-include requirements/app *.txt" + os.linesep,
-        "recursive-include src/lightning_app/cli/*-template *" + os.linesep,  # Add templates
-    ]
-
-    # TODO: remove this once lightning-ui package is ready as a dependency
-    lines += ["recursive-include src/lightning_app/ui *" + os.linesep]
-
-    with open(manifest_path, "w") as fp:
-        fp.writelines(lines)
-
-
-def _setup_args(**__: Any) -> Dict[str, Any]:
+def _setup_args() -> Dict[str, Any]:
     _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py")
     _setup_tools = _load_py_module("setup_tools", _path_setup_tools)
     _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))

src/lightning_lite/MANIFEST.in

@@ -0,0 +1,4 @@
+include src/lightning_lite/CHANGELOG.md
+include src/lightning_lite/README.md
+recursive-include requirements/lite *.txt
+include .actions/setup_tools.py

src/lightning_lite/__setup__.py

@@ -41,26 +41,7 @@ def _prepare_extras() -> Dict[str, Any]:
     return extras
 
 
-def _adjust_manifest(**__: Any) -> None:
-    manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in")
-    assert os.path.isfile(manifest_path)
-    with open(manifest_path) as fp:
-        lines = fp.readlines()
-    lines += [
-        "recursive-exclude src *.md" + os.linesep,
-        "recursive-exclude requirements *.txt" + os.linesep,
-        "recursive-include requirements/lite *.txt" + os.linesep,
-        "recursive-include src/lightning_lite *.md" + os.linesep,
-    ]
-
-    # TODO: remove this once lightning-ui package is ready as a dependency
-    lines += ["recursive-include src/lightning_app/ui *" + os.linesep]
-
-    with open(manifest_path, "w") as fp:
-        fp.writelines(lines)
-
-
-def _setup_args(**__: Any) -> Dict[str, Any]:
+def _setup_args() -> Dict[str, Any]:
     _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py")
     _setup_tools = _load_py_module("setup_tools", _path_setup_tools)
     _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))

src/pytorch_lightning/MANIFEST.in

@@ -0,0 +1,9 @@
+# distribute the lite source code inside PL
+include src/lightning_lite/CHANGELOG.md
+recursive-include requirements/lite *.txt
+include src/pytorch_lightning/CHANGELOG.md
+include src/pytorch_lightning/README.md
+recursive-include requirements/pytorch *.txt
+include .actions/setup_tools.py
+include *.cff  # citation info
+include src/pytorch_lightning/py.typed  # marker file for PEP 561

src/pytorch_lightning/__setup__.py

@@ -44,25 +44,7 @@ def _prepare_extras() -> Dict[str, Any]:
     return extras
 
 
-def _adjust_manifest(**__: Any) -> None:
-    manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in")
-    assert os.path.isfile(manifest_path)
-    with open(manifest_path) as fp:
-        lines = fp.readlines()
-    lines += [
-        "recursive-exclude src *.md" + os.linesep,
-        "recursive-exclude requirements *.txt" + os.linesep,
-        "recursive-include requirements/lite *.txt" + os.linesep,
-        "recursive-include src/lightning_lite *.md" + os.linesep,
-        "recursive-include src/pytorch_lightning *.md" + os.linesep,
-        "recursive-include requirements/pytorch *.txt" + os.linesep,
-        "include src/pytorch_lightning/py.typed" + os.linesep,  # marker file for PEP 561
-    ]
-    with open(manifest_path, "w") as fp:
-        fp.writelines(lines)
-
-
-def _setup_args(**__: Any) -> Dict[str, Any]:
+def _setup_args() -> Dict[str, Any]:
     _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py")
     _setup_tools = _load_py_module("setup_tools", _path_setup_tools)
     _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))