From 469941a52846ed3304210cf77f5c8a975b6ca6c3 Mon Sep 17 00:00:00 2001 From: Jiri BOROVEC Date: Mon, 5 Aug 2019 10:52:09 +0200 Subject: [PATCH] pkg relative imports * split requirements.txt * pytest verbose --- .readthedocs.yml | 2 +- .travis.yml | 3 ++- ...{doc_requirements.txt => requirements.txt} | 1 + .../multi_node_cluster_template.py | 12 +----------- .../single_cpu_template.py | 2 +- .../single_gpu_node_16bit_template.py | 2 +- .../single_gpu_node_ddp_template.py | 2 +- .../single_gpu_node_dp_template.py | 2 +- .../trainer_cpu_template.py | 3 ++- pytorch_lightning/callbacks/pt_callbacks.py | 7 ++++--- pytorch_lightning/models/trainer.py | 12 ++++++------ pytorch_lightning/root_module/grads.py | 3 +-- pytorch_lightning/root_module/memory.py | 12 ++++++------ pytorch_lightning/root_module/model_saving.py | 6 ++++-- pytorch_lightning/root_module/root_module.py | 11 ++++++----- pytorch_lightning/testing/lm_test_module.py | 19 ++++++++++--------- pytorch_lightning/utilities/arg_parse.py | 2 +- pytorch_lightning/utilities/debugging.py | 5 ++--- requirements.txt | 3 --- tests/requirements.txt | 2 ++ 20 files changed, 53 insertions(+), 58 deletions(-) rename docs/{doc_requirements.txt => requirements.txt} (63%) create mode 100644 tests/requirements.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index b85b4bc27fc08..fdb76cf621910 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -16,4 +16,4 @@ formats: all python: version: 3.7 install: - - requirements: docs/doc_requirements.txt \ No newline at end of file + - requirements: docs/requirements.txt \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index f4e08df63d144..d80b291f14299 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ cache: pip install: - pip install -e . - pip install -r requirements.txt + - pip install -r tests/requirements.txt - pip install -U numpy # keep build from timing out @@ -13,4 +14,4 @@ dist: xenial # command to run tests script: - - py.test # or py.test for Python versions 3.5 and below \ No newline at end of file + - py.test -v # or py.test for Python versions 3.5 and below \ No newline at end of file diff --git a/docs/doc_requirements.txt b/docs/requirements.txt similarity index 63% rename from docs/doc_requirements.txt rename to docs/requirements.txt index d52ea0ac4718e..4b1f1d920eb28 100644 --- a/docs/doc_requirements.txt +++ b/docs/requirements.txt @@ -1 +1,2 @@ mkdocs-material==4.4.0 +mkdocs==1.0.4 \ No newline at end of file diff --git a/examples/new_project_templates/multi_node_cluster_template.py b/examples/new_project_templates/multi_node_cluster_template.py index 88876ebe39036..5f6914d107ae0 100644 --- a/examples/new_project_templates/multi_node_cluster_template.py +++ b/examples/new_project_templates/multi_node_cluster_template.py @@ -14,17 +14,7 @@ torch.manual_seed(SEED) np.random.seed(SEED) -# --------------------- -# DEFINE MODEL HERE -# --------------------- -from lightning_module_template import LightningTemplateModel -# --------------------- - -""" -Allows training by using command line arguments -Run by: -# TYPE YOUR RUN COMMAND HERE -""" +from .lightning_module_template import LightningTemplateModel def main_local(hparams): diff --git a/examples/new_project_templates/single_cpu_template.py b/examples/new_project_templates/single_cpu_template.py index 2d1e8c2021c28..29c255981627a 100644 --- a/examples/new_project_templates/single_cpu_template.py +++ b/examples/new_project_templates/single_cpu_template.py @@ -13,7 +13,7 @@ torch.manual_seed(SEED) np.random.seed(SEED) -from lightning_module_template import LightningTemplateModel +from .lightning_module_template import LightningTemplateModel def main(hparams): diff --git a/examples/new_project_templates/single_gpu_node_16bit_template.py b/examples/new_project_templates/single_gpu_node_16bit_template.py index 7ece5adef6be1..14db484e56409 100644 --- a/examples/new_project_templates/single_gpu_node_16bit_template.py +++ b/examples/new_project_templates/single_gpu_node_16bit_template.py @@ -13,7 +13,7 @@ torch.manual_seed(SEED) np.random.seed(SEED) -from lightning_module_template import LightningTemplateModel +from .lightning_module_template import LightningTemplateModel def main(hparams): diff --git a/examples/new_project_templates/single_gpu_node_ddp_template.py b/examples/new_project_templates/single_gpu_node_ddp_template.py index 6d10062dd5abe..56e301b20478a 100644 --- a/examples/new_project_templates/single_gpu_node_ddp_template.py +++ b/examples/new_project_templates/single_gpu_node_ddp_template.py @@ -13,7 +13,7 @@ torch.manual_seed(SEED) np.random.seed(SEED) -from lightning_module_template import LightningTemplateModel +from .lightning_module_template import LightningTemplateModel def main(hparams): diff --git a/examples/new_project_templates/single_gpu_node_dp_template.py b/examples/new_project_templates/single_gpu_node_dp_template.py index 50c615ad8f2a4..9d6992533b680 100644 --- a/examples/new_project_templates/single_gpu_node_dp_template.py +++ b/examples/new_project_templates/single_gpu_node_dp_template.py @@ -13,7 +13,7 @@ torch.manual_seed(SEED) np.random.seed(SEED) -from lightning_module_template import LightningTemplateModel +from .lightning_module_template import LightningTemplateModel def main(hparams): diff --git a/examples/new_project_templates/trainer_cpu_template.py b/examples/new_project_templates/trainer_cpu_template.py index 97dbfba74a66b..de6ba7c424571 100644 --- a/examples/new_project_templates/trainer_cpu_template.py +++ b/examples/new_project_templates/trainer_cpu_template.py @@ -5,7 +5,8 @@ from pytorch_lightning.models.trainer import Trainer from pytorch_lightning.utilities.arg_parse import add_default_args from pytorch_lightning.callbacks.pt_callbacks import EarlyStopping, ModelCheckpoint -from lightning_module_template import LightningTemplateModel + +from .lightning_module_template import LightningTemplateModel def main(hparams): diff --git a/pytorch_lightning/callbacks/pt_callbacks.py b/pytorch_lightning/callbacks/pt_callbacks.py index af5cbb42be9f2..89c8f6b2e5f8f 100644 --- a/pytorch_lightning/callbacks/pt_callbacks.py +++ b/pytorch_lightning/callbacks/pt_callbacks.py @@ -1,7 +1,9 @@ -import numpy as np import os import shutil -from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel + +import numpy as np + +from ..pt_overrides.override_data_parallel import LightningDistributedDataParallel class Callback(object): @@ -261,4 +263,3 @@ def on_epoch_end(self, epoch, logs=None): print(loss) if should_stop: break - diff --git a/pytorch_lightning/models/trainer.py b/pytorch_lightning/models/trainer.py index 098526a85ab12..858b8e5d4caa6 100644 --- a/pytorch_lightning/models/trainer.py +++ b/pytorch_lightning/models/trainer.py @@ -8,17 +8,17 @@ import pdb import re +import numpy as np +import tqdm import torch from torch.utils.data.distributed import DistributedSampler import torch.multiprocessing as mp import torch.distributed as dist -import numpy as np -import tqdm -from pytorch_lightning.root_module.memory import get_gpu_memory_map -from pytorch_lightning.root_module.model_saving import TrainerIO -from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel, LightningDataParallel -from pytorch_lightning.utilities.debugging import MisconfigurationException +from ..root_module.memory import get_gpu_memory_map +from ..root_module.model_saving import TrainerIO +from ..pt_overrides.override_data_parallel import LightningDistributedDataParallel, LightningDataParallel +from ..utilities.debugging import MisconfigurationException try: from apex import amp diff --git a/pytorch_lightning/root_module/grads.py b/pytorch_lightning/root_module/grads.py index e368bb5f501f9..d50fa450961a2 100644 --- a/pytorch_lightning/root_module/grads.py +++ b/pytorch_lightning/root_module/grads.py @@ -1,9 +1,8 @@ -from torch import nn - """ Module to describe gradients """ +from torch import nn class GradInformation(nn.Module): diff --git a/pytorch_lightning/root_module/memory.py b/pytorch_lightning/root_module/memory.py index 389c8680ac1b3..128fb16f18b43 100644 --- a/pytorch_lightning/root_module/memory.py +++ b/pytorch_lightning/root_module/memory.py @@ -1,15 +1,15 @@ -import torch +''' +Generates a summary of a model's layers and dimensionality +''' + import gc + +import torch import subprocess import numpy as np import pandas as pd -''' -Generates a summary of a model's layers and dimensionality -''' - - class ModelSummary(object): def __init__(self, model): diff --git a/pytorch_lightning/root_module/model_saving.py b/pytorch_lightning/root_module/model_saving.py index 34e4485b7e55f..0bde0943f45a8 100644 --- a/pytorch_lightning/root_module/model_saving.py +++ b/pytorch_lightning/root_module/model_saving.py @@ -1,7 +1,9 @@ -import torch import os import re -from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel, LightningDataParallel + +import torch + +from ..pt_overrides.override_data_parallel import LightningDistributedDataParallel, LightningDataParallel class ModelIO(object): diff --git a/pytorch_lightning/root_module/root_module.py b/pytorch_lightning/root_module/root_module.py index c860685eea869..1421a72d5b357 100644 --- a/pytorch_lightning/root_module/root_module.py +++ b/pytorch_lightning/root_module/root_module.py @@ -1,9 +1,10 @@ import torch -from pytorch_lightning.root_module.memory import ModelSummary -from pytorch_lightning.root_module.grads import GradInformation -from pytorch_lightning.root_module.model_saving import ModelIO, load_hparams_from_tags_csv -from pytorch_lightning.root_module.hooks import ModelHooks -from pytorch_lightning.root_module.decorators import data_loader + +from .memory import ModelSummary +from .grads import GradInformation +from .model_saving import ModelIO, load_hparams_from_tags_csv +from .hooks import ModelHooks +from .decorators import data_loader class LightningModule(GradInformation, ModelIO, ModelHooks): diff --git a/pytorch_lightning/testing/lm_test_module.py b/pytorch_lightning/testing/lm_test_module.py index 8e0718b972263..9861810e8a7f8 100644 --- a/pytorch_lightning/testing/lm_test_module.py +++ b/pytorch_lightning/testing/lm_test_module.py @@ -1,17 +1,18 @@ import os from collections import OrderedDict -import torch.nn as nn -from torchvision.datasets import MNIST -import torchvision.transforms as transforms + import torch +import torch.nn as nn import torch.nn.functional as F -from test_tube import HyperOptArgumentParser from torch import optim from torch.utils.data import DataLoader from torch.utils.data.distributed import DistributedSampler +from torchvision.datasets import MNIST +from torchvision import transforms +from test_tube import HyperOptArgumentParser -from pytorch_lightning.root_module.root_module import LightningModule -import pytorch_lightning as ptl +from ..root_module.root_module import LightningModule +from pytorch_lightning import data_loader class LightningTestModel(LightningModule): @@ -214,15 +215,15 @@ def __dataloader(self, train): return loader - @ptl.data_loader + @data_loader def tng_dataloader(self): return self.__dataloader(train=True) - @ptl.data_loader + @data_loader def val_dataloader(self): return self.__dataloader(train=False) - @ptl.data_loader + @data_loader def test_dataloader(self): return self.__dataloader(train=False) diff --git a/pytorch_lightning/utilities/arg_parse.py b/pytorch_lightning/utilities/arg_parse.py index 72e38686cb125..91696433fe83e 100644 --- a/pytorch_lightning/utilities/arg_parse.py +++ b/pytorch_lightning/utilities/arg_parse.py @@ -1,4 +1,4 @@ -import pdb + def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None): diff --git a/pytorch_lightning/utilities/debugging.py b/pytorch_lightning/utilities/debugging.py index 3091ff3ba9279..5eacc240a8946 100644 --- a/pytorch_lightning/utilities/debugging.py +++ b/pytorch_lightning/utilities/debugging.py @@ -1,5 +1,4 @@ -import pdb -import sys + class MisconfigurationException(Exception): - pass \ No newline at end of file + pass diff --git a/requirements.txt b/requirements.txt index 3e863f05ba6d9..aba573c0c6085 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,3 @@ -coverage==4.5.3 -mkdocs==1.0.4 -pytest==5.0.1 scikit-learn==0.20.2 tqdm==4.32.1 twine==1.13.0 diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000000000..c16efc2add252 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,2 @@ +coverage==4.5.3 +pytest==5.0.1