diff --git a/.all-contributorsrc b/.all-contributorsrc new file mode 100644 index 000000000000..7a7e4cf62e5e --- /dev/null +++ b/.all-contributorsrc @@ -0,0 +1,55 @@ +{ + "projectName": "edflow", + "projectOwner": "PEsser", + "repoType": "github", + "repoHost": "https://github.com", + "files": ["README.md"], + "imageSize": 100, + "commit": false, + "commitConvention": "none", + "contributors": [ + { + "login": "pesser", + "name": "Patrick Esser", + "avatar_url": "https://avatars3.githubusercontent.com/u/2175508?v=4", + "profile": "https://github.com/pesser", + "contributions": ["code"], + }, + { + "login": "jhaux", + "name": "Johannes Haux", + "avatar_url": "https://avatars0.githubusercontent.com/u/9572598?v=4", + "profile": "https://github.com/jhaux", + "contributions": ["code"], + }, + { + "login": "ArWeHei", + "name": "arwehei", + "avatar_url": "https://avatars2.githubusercontent.com/u/46443020?v=4", + "profile": "https://github.com/ArWeHei", + "contributions": ["doc"], + }, + { + "login": "theRealSuperMario", + "name": "therealsupermario", + "avatar_url": "https://avatars0.githubusercontent.com/u/6517465?v=4", + "profile": "https://github.com/theRealSuperMario", + "contributions": ["example"], + }, + { + "login": "mritv", + "name": "Ritvik Marwaha", + "avatar_url": "https://avatars1.githubusercontent.com/u/39053439?v=4", + "profile": "https://github.com/mritv", + "contributions": ["example"], + }, + { + "login": "conrad784", + "name": "Conrad Sachweh", + "avatar_url": "https://avatars0.githubusercontent.com/u/6422533?v=4", + "profile": "https://conrad-sachweh.de", + "contributions": ["test"], + }, + ], + "contributorsPerLine": 7, +} diff --git a/README.md b/README.md index 4adfe680917b..bfb6e14e91bf 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ where `` is one of the `yaml` files described above. ## Workflow -For more information, look into our [starter guide](link). +For more information, look into our [documentation](https://edflow.readthedocs.io/en/latest/). ## Example @@ -104,3 +104,16 @@ Mimo Tilbich [![GitHub-Contributions][GitHub-Contributions]](https://github.com/ [GitHub-Updated]: https://img.shields.io/github/last-commit/pesser/edflow/master.svg?logo=github&logoColor=white&label=pushed [LICENSE]: https://img.shields.io/github/license/pesser/edflow.svg + +## Contributors + +Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): + + + +
Patrick Esser
Patrick Esser
💻		Johannes Haux
Johannes Haux
💻		arwehei
arwehei
📖		therealsupermario
therealsupermario
💡		Ritvik Marwaha
Ritvik Marwaha
💡		Conrad Sachweh
Conrad Sachweh
⚠️
+ + + +This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! + source/source_files/edflow diff --git a/docs/conf.py b/docs/conf.py index b548207a3753..a52ef76a85f2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,7 +38,6 @@ "sphinx.ext.ifconfig", "sphinx.ext.viewcode", "sphinx.ext.githubpages", - "sphinx.ext.autosummary", ] # Add any paths that contain templates here, relative to this directory. @@ -80,12 +79,11 @@ "natsort", ] -autosummary_generate = True -exclude_patterns = ["_build", "_templates"] - autodoc_default_options = { "members": True, "member-order": "bysource", "special-members": "__init__", "undoc-members": True, } + +exclude_patterns = ["_build", "_templates"] diff --git a/docs/index.rst b/docs/index.rst index e0df0dbbb3b0..8596b651267c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,7 @@ We created a user guide for you to use, have a look: Contents ======== .. toctree:: + :glob: :maxdepth: 2 source/intro @@ -28,7 +29,7 @@ Contents source/models source/iterator source/contributions - source/docu + source/source_files/edflow Indices and tables ================== diff --git a/docs/source/contributions.rst b/docs/source/contributions.rst index 40eb1ad50a90..416efbbff335 100644 --- a/docs/source/contributions.rst +++ b/docs/source/contributions.rst @@ -1,6 +1,5 @@ - Contributions -============= +************* If you have any new applications that require custom hooks or iterators feel free to contribute at any time. @@ -9,7 +8,90 @@ free to contribute at any time. Examples of models are always welcome and we are happy if want to contribute in any way. -We are working on github and celebrate every pull request ;). +We are working on github and celebrate every pull request. + +black +----- + +Before requesting a pull please run black_ for better code style or simply add +black_ to your pre-commit hook: + +0. Install black_ with +:: + $ pip install black +1. Paste the following into at the top /.git/hooks/pre-commit.sample +:: + # run black on all staged files + staged=$(git diff --name-only --cached) + black $staged + # add them again after formatting + git add $staged +2. Rename ``pre-commit.sample`` to ``pre-commit`` +3. Make it excutable using +:: + $ chmod +x pre-commit +4. Done! + +Or run black by hand and use this command before every commit: +:: + black ./ + +Continuous Integration +====================== + +We use travisCI_ for continuous integration. +You do not need to worry about as long as your code passes all tests (this includes +a formatting test with black). + +Documenation +============ + + This is a short summary how the documentation works and how it can be built + +The documentation uses sphinx_ and is available under readthedocs.org_. +It also uses all-contributors_ for honoring contributors. + +sphinx +------ + +To build the documentation locally, install `sphinx` and run: +:: + $ cd docs + $ make html + +The html files are available under the then existing direcotry ``docs/_build/html/`` + +The docsting format which is preferred is `numpy`. + +We use `sphinx-apidoc` to track all files automatically: +:: + $ cd docs + $ sphinx-apidoc -o ./source/source_files ../edflow + +all-contirbutors +---------------- + +We use all-contributors locally and manage the contributors by hand. + +To do so, install `all-contributors` as described here (we advise you to install it inside the repo but unstage the added files). +Then run the following command to add a contributor or contribution: +:: + all-contributors add + +If this does not work for you (sometimes with npm the case) use: +:: + ./node_modules/.bin/all-contributors add + +Known Issues +------------ + +We noticed that mocking `numpy` in ``config.py`` will not work due to some requirements when importing numpy in edflow. +Thus we need to require numpy when building the documentation. + +Locally, this means that you need to have numpy installed in your environment. + +Concerning ``readthedocs.org``, this means that we require a ``readthedocs.yml`` in the source directory which points to ``extra_requirements`` in ``setup.py``, where numpy is a dependency. +Other dependencies are `sphinx` and `sphinx_rtd_theme`. .. image:: https://img.shields.io/github/commit-activity/y/pesser/edflow.svg?logo=github&logoColor=white @@ -41,3 +123,11 @@ We are working on github and celebrate every pull request ;). :target: https://img.shields.io/github/forks/pesser/edflow.svg?logo=github&logoColor=white :alt: GitHub-Forks + +.. _black: https://github.com/ambv/black + +.. _readthedocs.org: https://edflow.readthedocs.io/en/latest/ + +.. _all-contributors: https://allcontributors.org + +.. _travisCI: https://travis-ci.org/pesser/edflow/ diff --git a/docs/source/docu.rst b/docs/source/docu.rst deleted file mode 100644 index 453c804b947e..000000000000 --- a/docs/source/docu.rst +++ /dev/null @@ -1,31 +0,0 @@ - -Documentation -============= - -.. autosummary:: - - edflow.custom_logging - edflow.debug - edflow.evaluate - edflow.main - edflow.model - edflow.project_manager - edflow.train - edflow.util - edflow.data.dataset - edflow.data.util - edflow.hooks.evaluation_hooks - edflow.hooks.hook - edflow.hooks.pytorch_hooks - edflow.hooks.runtime_input - edflow.hooks.train_hooks - edflow.hooks.util_hooks - edflow.iterators.batches - edflow.iterators.deeploss - edflow.iterators.evaluator - edflow.iterators.model_iterator - edflow.iterators.resize - edflow.iterators.trainer - edflow.metrics.image_metrics - - diff --git a/docs/source/iterator.rst b/docs/source/iterator.rst index 30d919f51af7..0300c62da27f 100644 --- a/docs/source/iterator.rst +++ b/docs/source/iterator.rst @@ -1,6 +1,6 @@ -Documentation -============= +Iterators +========= PyIterator ---------- diff --git a/docs/source/run_edflow.rst b/docs/source/run_edflow.rst index 85db36d6a2a0..dafac23f3aa2 100644 --- a/docs/source/run_edflow.rst +++ b/docs/source/run_edflow.rst @@ -1,6 +1,6 @@ What Happens When I Run ``EDflow`` -================================== +********************************** When you successfully build your model start ``EDflow`` with:: edflow -t your_model/train.yaml diff --git a/docs/source/source_files/edflow.applications.rst b/docs/source/source_files/edflow.applications.rst new file mode 100644 index 000000000000..93f64d06f7da --- /dev/null +++ b/docs/source/source_files/edflow.applications.rst @@ -0,0 +1,22 @@ +edflow.applications package +=========================== + +Submodules +---------- + +edflow.applications.tf\_perceptual\_loss module +----------------------------------------------- + +.. automodule:: edflow.applications.tf_perceptual_loss + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: edflow.applications + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/source_files/edflow.data.rst b/docs/source/source_files/edflow.data.rst new file mode 100644 index 000000000000..9cfb95c1c473 --- /dev/null +++ b/docs/source/source_files/edflow.data.rst @@ -0,0 +1,30 @@ +edflow.data package +=================== + +Submodules +---------- + +edflow.data.dataset module +-------------------------- + +.. automodule:: edflow.data.dataset + :members: + :undoc-members: + :show-inheritance: + +edflow.data.util module +----------------------- + +.. automodule:: edflow.data.util + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: edflow.data + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/source_files/edflow.hooks.checkpoint_hooks.rst b/docs/source/source_files/edflow.hooks.checkpoint_hooks.rst new file mode 100644 index 000000000000..41239bccc6fd --- /dev/null +++ b/docs/source/source_files/edflow.hooks.checkpoint_hooks.rst @@ -0,0 +1,38 @@ +edflow.hooks.checkpoint\_hooks package +====================================== + +Submodules +---------- + +edflow.hooks.checkpoint\_hooks.common module +-------------------------------------------- + +.. automodule:: edflow.hooks.checkpoint_hooks.common + :members: + :undoc-members: + :show-inheritance: + +edflow.hooks.checkpoint\_hooks.tf\_checkpoint\_hook module +---------------------------------------------------------- + +.. automodule:: edflow.hooks.checkpoint_hooks.tf_checkpoint_hook + :members: + :undoc-members: + :show-inheritance: + +edflow.hooks.checkpoint\_hooks.torch\_checkpoint\_hook module +------------------------------------------------------------- + +.. automodule:: edflow.hooks.checkpoint_hooks.torch_checkpoint_hook + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: edflow.hooks.checkpoint_hooks + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/source_files/edflow.hooks.logging_hooks.rst b/docs/source/source_files/edflow.hooks.logging_hooks.rst new file mode 100644 index 000000000000..310f5a56a568 --- /dev/null +++ b/docs/source/source_files/edflow.hooks.logging_hooks.rst @@ -0,0 +1,22 @@ +edflow.hooks.logging\_hooks package +=================================== + +Submodules +---------- + +edflow.hooks.logging\_hooks.tf\_logging\_hook module +---------------------------------------------------- + +.. automodule:: edflow.hooks.logging_hooks.tf_logging_hook + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: edflow.hooks.logging_hooks + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/source_files/edflow.hooks.metric_hooks.rst b/docs/source/source_files/edflow.hooks.metric_hooks.rst new file mode 100644 index 000000000000..4e7d17331b71 --- /dev/null +++ b/docs/source/source_files/edflow.hooks.metric_hooks.rst @@ -0,0 +1,22 @@ +edflow.hooks.metric\_hooks package +================================== + +Submodules +---------- + +edflow.hooks.metric\_hooks.tf\_metric\_hook module +-------------------------------------------------- + +.. automodule:: edflow.hooks.metric_hooks.tf_metric_hook + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: edflow.hooks.metric_hooks + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/source_files/edflow.hooks.rst b/docs/source/source_files/edflow.hooks.rst new file mode 100644 index 000000000000..2efcecbe597d --- /dev/null +++ b/docs/source/source_files/edflow.hooks.rst @@ -0,0 +1,55 @@ +edflow.hooks package +==================== + +Subpackages +----------- + +.. toctree:: + + edflow.hooks.checkpoint_hooks + edflow.hooks.logging_hooks + edflow.hooks.metric_hooks + +Submodules +---------- + +edflow.hooks.hook module +------------------------ + +.. automodule:: edflow.hooks.hook + :members: + :undoc-members: + :show-inheritance: + +edflow.hooks.pytorch\_hooks module +---------------------------------- + +.. automodule:: edflow.hooks.pytorch_hooks + :members: + :undoc-members: + :show-inheritance: + +edflow.hooks.runtime\_input module +---------------------------------- + +.. automodule:: edflow.hooks.runtime_input + :members: + :undoc-members: + :show-inheritance: + +edflow.hooks.util\_hooks module +------------------------------- + +.. automodule:: edflow.hooks.util_hooks + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: edflow.hooks + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/source_files/edflow.iterators.rst b/docs/source/source_files/edflow.iterators.rst new file mode 100644 index 000000000000..57e6b93e0b29 --- /dev/null +++ b/docs/source/source_files/edflow.iterators.rst @@ -0,0 +1,70 @@ +edflow.iterators package +======================== + +Submodules +---------- + +edflow.iterators.batches module +------------------------------- + +.. automodule:: edflow.iterators.batches + :members: + :undoc-members: + :show-inheritance: + +edflow.iterators.model\_iterator module +--------------------------------------- + +.. automodule:: edflow.iterators.model_iterator + :members: + :undoc-members: + :show-inheritance: + +edflow.iterators.resize module +------------------------------ + +.. automodule:: edflow.iterators.resize + :members: + :undoc-members: + :show-inheritance: + +edflow.iterators.tf\_evaluator module +------------------------------------- + +.. automodule:: edflow.iterators.tf_evaluator + :members: + :undoc-members: + :show-inheritance: + +edflow.iterators.tf\_iterator module +------------------------------------ + +.. automodule:: edflow.iterators.tf_iterator + :members: + :undoc-members: + :show-inheritance: + +edflow.iterators.tf\_trainer module +----------------------------------- + +.. automodule:: edflow.iterators.tf_trainer + :members: + :undoc-members: + :show-inheritance: + +edflow.iterators.torch\_iterator module +--------------------------------------- + +.. automodule:: edflow.iterators.torch_iterator + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: edflow.iterators + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/source_files/edflow.metrics.rst b/docs/source/source_files/edflow.metrics.rst new file mode 100644 index 000000000000..0e0c8721a52d --- /dev/null +++ b/docs/source/source_files/edflow.metrics.rst @@ -0,0 +1,22 @@ +edflow.metrics package +====================== + +Submodules +---------- + +edflow.metrics.image\_metrics module +------------------------------------ + +.. automodule:: edflow.metrics.image_metrics + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: edflow.metrics + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/source_files/edflow.rst b/docs/source/source_files/edflow.rst new file mode 100644 index 000000000000..bce18a99fdbf --- /dev/null +++ b/docs/source/source_files/edflow.rst @@ -0,0 +1,73 @@ +edflow package +============== + +Subpackages +----------- + +.. toctree:: + + edflow.applications + edflow.data + edflow.hooks + edflow.iterators + edflow.metrics + +Submodules +---------- + +edflow.custom\_logging module +----------------------------- + +.. automodule:: edflow.custom_logging + :members: + :undoc-members: + :show-inheritance: + +edflow.debug module +------------------- + +.. automodule:: edflow.debug + :members: + :undoc-members: + :show-inheritance: + +edflow.main module +------------------ + +.. automodule:: edflow.main + :members: + :undoc-members: + :show-inheritance: + +edflow.project\_manager module +------------------------------ + +.. automodule:: edflow.project_manager + :members: + :undoc-members: + :show-inheritance: + +edflow.tf\_util module +---------------------- + +.. automodule:: edflow.tf_util + :members: + :undoc-members: + :show-inheritance: + +edflow.util module +------------------ + +.. automodule:: edflow.util + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: edflow + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/source_files/modules.rst b/docs/source/source_files/modules.rst new file mode 100644 index 000000000000..be5f3efe8760 --- /dev/null +++ b/docs/source/source_files/modules.rst @@ -0,0 +1,7 @@ +edflow +====== + +.. toctree:: + :maxdepth: 4 + + edflow diff --git a/edflow/hooks/checkpoint_hooks/torch_checkpoint_hook.py b/edflow/hooks/checkpoint_hooks/torch_checkpoint_hook.py index 28ad192a80b7..37ee430fcf5b 100644 --- a/edflow/hooks/checkpoint_hooks/torch_checkpoint_hook.py +++ b/edflow/hooks/checkpoint_hooks/torch_checkpoint_hook.py @@ -1,5 +1,7 @@ import torch +from edflow.hooks.hook import Hook + class RestorePytorchModelHook(Hook): """Restores a PyTorch model from a checkpoint at each epoch. Can also be diff --git a/edflow/hooks/metric_hooks/tf_metric_hook.py b/edflow/hooks/metric_hooks/tf_metric_hook.py index 4ba523d62eb1..07411f13e922 100644 --- a/edflow/hooks/metric_hooks/tf_metric_hook.py +++ b/edflow/hooks/metric_hooks/tf_metric_hook.py @@ -1,3 +1,6 @@ +from edflow.hooks.hook import Hook + + class MetricHook(Hook): """Applies a set of given metrics to the calculated data.""" diff --git a/edflow/iterators/model_iterator.py b/edflow/iterators/model_iterator.py index 02d1250baa02..0d79365cfc41 100644 --- a/edflow/iterators/model_iterator.py +++ b/edflow/iterators/model_iterator.py @@ -161,13 +161,14 @@ def run_hooks( fetches (list or dict): Fetches for the next session.run call. feeds (dict): Feeds for the next session.run call. results (same as fetches): Results from the last session.run call. - before (bool): If not obvious determines if the before_ or after_ + before (bool): If not obvious determines if the before or after methods of the hooks should be called. - Return: + Returns: If before: - same as fetches: Updated fetches. - dict: Updated feeds + + test (same as fetches): Updated fetches. + test (dict): Updated feeds """ is_step = fetches is not None and feeds is not None diff --git a/edflow/iterators/tf_evaluator.py b/edflow/iterators/tf_evaluator.py index 060a73aa5c8d..64d877349dd4 100644 --- a/edflow/iterators/tf_evaluator.py +++ b/edflow/iterators/tf_evaluator.py @@ -1,9 +1,12 @@ import tensorflow as tf import os -from edflow.iterators.model_iterator import HookedModelIterator, TFHookedModelIterator -from edflow.hooks import WaitForCheckpointHook, RestoreModelHook -from edflow.hooks.evaluation_hooks import RestoreTFModelHook +from edflow.iterators.tf_iterator import HookedModelIterator, TFHookedModelIterator +from edflow.hooks.checkpoint_hooks.common import WaitForCheckpointHook +from edflow.hooks.checkpoint_hooks.tf_checkpoint_hook import ( + RestoreModelHook, + RestoreTFModelHook, +) from edflow.project_manager import ProjectManager diff --git a/setup.py b/setup.py index 44a11ad9b43b..1d8133757229 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ "psutil", "pytest", ], - extras_require={"docs": ["sphinx >= 1.4", "sphinx_rtd_theme"]}, + extras_require={"docs": ["sphinx >= 1.4", "sphinx_rtd_theme", "numpy"]}, zip_safe=False, scripts=["edflow/edflow", "edflow/edcache", "edflow/edlist"], )