From b912fd6e85135ab8e708d6bceeb95b9c37432f6f Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Mon, 10 Jan 2022 09:49:18 +0100 Subject: [PATCH 01/14] source /home/ashwin/miniconda3/bin/activate# This is a combination of 2 commits. conda activate anomalib# This is the 1st commit message: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🚀 Add nightly build to CI 🚚 Move PR template to .github root --- .../pull_request_template.md | 2 +- .github/workflows/nightly.yml | 31 +++++++++++++++++++ .github/workflows/tox.yml | 1 + tests/models/test_model.py | 20 +++++++++--- tox.ini | 1 + 5 files changed, 49 insertions(+), 6 deletions(-) rename .github/{PULL_REQUEST_TEMPLATE => }/pull_request_template.md (98%) create mode 100644 .github/workflows/nightly.yml diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/pull_request_template.md similarity index 98% rename from .github/PULL_REQUEST_TEMPLATE/pull_request_template.md rename to .github/pull_request_template.md index 16d09bcdfa..54e6d7e3f1 100644 --- a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,7 +11,7 @@ - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] This change requires a documentation update -# Checklist: +## Checklist - [ ] My code follows the [pre-commit style and check guidelines](https://openvinotoolkit.github.io/anomalib/guides/using_pre_commit.html#pre-commit-hooks) of this project. - [ ] I have performed a self-review of my code diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000000..3646328555 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,31 @@ +name: Nightly-regression Test + +on: + workflow_dispatch: # run on request (no need for PR) + schedule: + - cron '0 0 * * *' + +jobs: + Tox: + runs-on: [self-hosted, linux, x64] + strategy: + max-parallel: 1 + if: github.ref == 'refs/heads/development' + steps: + - name: Print GPU status + run: nvidia-smi + - name: CHECKOUT REPOSITORY + uses: actions/checkout@v2 + - name: Install Tox + run: pip install tox + - name: Coverage + run: | + export ANOMALIB_DATASET_PATH=/media/data1/datasets/MVTec + export CUDA_VISIBLE_DEVICES=2 + export NIGHTLY_BUILD=TRUE + tox -e coverage + - name: Upload coverage result + uses: actions/upload-artifact@v2 + with: + name: coverage + path: .tox/coverage.xml diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 3a44b136cb..8f1359c0e1 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -24,6 +24,7 @@ jobs: run: | export ANOMALIB_DATASET_PATH=/media/data1/datasets/MVTec export CUDA_VISIBLE_DEVICES=3 + export NIGHTLY_BUILD=FALSE tox -e coverage - name: Upload coverage result uses: actions/upload-artifact@v2 diff --git a/tests/models/test_model.py b/tests/models/test_model.py index 9b11c57455..e20abe3eb3 100644 --- a/tests/models/test_model.py +++ b/tests/models/test_model.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions # and limitations under the License. +import os import random import tempfile from functools import wraps @@ -30,6 +31,14 @@ from tests.helpers.dataset import TestDataset, get_dataset_path +@pytest.fixture(autouse=True) +def nightly_build() -> bool: + """If NIGHTLY_BUILD env variable is set to TRUE, the tests will use mvtec and check performance.""" + if os.environ["NIGHTLY_BUILD"] == "TRUE": + return True + return False + + @pytest.fixture(autouse=True) def category() -> str: """PyTest fixture to randomly return an MVTec category. @@ -117,10 +126,11 @@ def _test_metrics(self, trainer, config, model, datamodule): results = trainer.test(model=model, datamodule=datamodule)[0] - assert results["image_AUROC"] >= 0.6 + if nightly_build: + assert results["image_AUROC"] >= 0.6 - if config.dataset.task == "segmentation": - assert results["pixel_AUROC"] >= 0.6 + if config.dataset.task == "segmentation": + assert results["pixel_AUROC"] >= 0.6 return results def _test_model_load(self, config, datamodule, results): @@ -159,9 +169,9 @@ def _test_model_load(self, config, datamodule, results): ], ) @pytest.mark.flaky(max_runs=3) - @TestDataset(num_train=200, num_test=10, path=get_dataset_path(), use_mvtec=True) + @TestDataset(num_train=200, num_test=10, path=get_dataset_path(), use_mvtec=nightly_build) @AddDFMScores() - def test_model(self, category, model_name, nncf, use_mvtec=True, path="./datasets/MVTec", score_type=None): + def test_model(self, category, model_name, nncf, use_mvtec=nightly_build, path="./datasets/MVTec", score_type=None): """Driver for all the tests in the class.""" with tempfile.TemporaryDirectory() as project_path: model, config, datamodule, trainer = self._setup( diff --git a/tox.ini b/tox.ini index e80af51c8f..6543da9c75 100644 --- a/tox.ini +++ b/tox.ini @@ -64,6 +64,7 @@ passenv = ftp_proxy HTTPS_PROXY CUDA_VISIBLE_DEVICES ANOMALIB_DATASET_PATH + NIGHTLY_BUILD deps = coverage pytest From 636788ccd72ca929541b9d824759e9db7ed45950 Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Mon, 10 Jan 2022 09:57:58 +0100 Subject: [PATCH 02/14] =?UTF-8?q?=F0=9F=9A=80=20Fix=20syntax?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3646328555..519d3ff691 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,7 +3,7 @@ name: Nightly-regression Test on: workflow_dispatch: # run on request (no need for PR) schedule: - - cron '0 0 * * *' + - cron: "0 0 * * *" jobs: Tox: From 9bcfb7c795ea97088366ca0e3d7a38d692edbcf4 Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Mon, 24 Jan 2022 15:00:29 +0100 Subject: [PATCH 03/14] =?UTF-8?q?=F0=9F=94=A8=20Refactor=20model=20train?= =?UTF-8?q?=20tests=20into=20pre-merge=20and=20nightly.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/{tox.yml => pre_merge.yml} | 0 anomalib/core/callbacks/model_loader.py | 2 +- anomalib/core/metrics/auroc.py | 2 +- anomalib/utils/hpo/__init__.py | 15 ++ anomalib/utils/hpo/config.py | 53 +++++ tests/helpers/model.py | 126 ++++++++++++ tests/models/test_model.py | 191 ------------------ .../deploy}/__init__.py | 0 tests/{ => nightly}/deploy/test_inferencer.py | 4 + .../models}/__init__.py | 0 tests/nightly/models/test_model.py | 130 ++++++++++++ tests/{ => pre_merge}/config/__init__.py | 0 tests/{ => pre_merge}/config/test_config.py | 0 .../callbacks/compress_callback}/__init__.py | 0 .../compress_callback/dummy_config.yml | 0 .../dummy_lightning_model.py | 0 .../compress_callback/test_compress.py | 6 +- .../normalization_callback}/__init__.py | 0 .../test_normalization_callback.py | 0 .../visualizer_callback}/__init__.py | 0 .../dummy_lightning_model.py | 0 .../visualizer_callback/test_visualizer.py | 0 tests/pre_merge/datasets/__init__.py | 15 ++ .../{ => pre_merge}/datasets/dummy_config.yml | 0 .../{ => pre_merge}/datasets/test_dataset.py | 0 tests/{ => pre_merge}/datasets/test_tiler.py | 0 .../datasets/test_transforms.py | 0 tests/{ => pre_merge}/loggers/__init__.py | 0 .../loggers/test_get_logger.py | 0 tests/{ => pre_merge}/models/__init__.py | 2 +- tests/pre_merge/models/test_model.py | 67 ++++++ tests/{ => pre_merge}/utils/__init__.py | 0 .../utils/test_download_progress_bar.py | 0 33 files changed, 417 insertions(+), 196 deletions(-) rename .github/workflows/{tox.yml => pre_merge.yml} (100%) create mode 100644 anomalib/utils/hpo/__init__.py create mode 100644 anomalib/utils/hpo/config.py create mode 100644 tests/helpers/model.py delete mode 100644 tests/models/test_model.py rename tests/{core/callbacks/compress_callback => nightly/deploy}/__init__.py (100%) rename tests/{ => nightly}/deploy/test_inferencer.py (97%) rename tests/{core/callbacks/normalization_callback => nightly/models}/__init__.py (100%) create mode 100644 tests/nightly/models/test_model.py rename tests/{ => pre_merge}/config/__init__.py (100%) rename tests/{ => pre_merge}/config/test_config.py (100%) rename tests/{core/callbacks/visualizer_callback => pre_merge/core/callbacks/compress_callback}/__init__.py (100%) rename tests/{ => pre_merge}/core/callbacks/compress_callback/dummy_config.yml (100%) rename tests/{ => pre_merge}/core/callbacks/compress_callback/dummy_lightning_model.py (100%) rename tests/{ => pre_merge}/core/callbacks/compress_callback/test_compress.py (84%) rename tests/{datasets => pre_merge/core/callbacks/normalization_callback}/__init__.py (100%) rename tests/{ => pre_merge}/core/callbacks/normalization_callback/test_normalization_callback.py (100%) rename tests/{deploy => pre_merge/core/callbacks/visualizer_callback}/__init__.py (100%) rename tests/{ => pre_merge}/core/callbacks/visualizer_callback/dummy_lightning_model.py (100%) rename tests/{ => pre_merge}/core/callbacks/visualizer_callback/test_visualizer.py (100%) create mode 100644 tests/pre_merge/datasets/__init__.py rename tests/{ => pre_merge}/datasets/dummy_config.yml (100%) rename tests/{ => pre_merge}/datasets/test_dataset.py (100%) rename tests/{ => pre_merge}/datasets/test_tiler.py (100%) rename tests/{ => pre_merge}/datasets/test_transforms.py (100%) rename tests/{ => pre_merge}/loggers/__init__.py (100%) rename tests/{ => pre_merge}/loggers/test_get_logger.py (100%) rename tests/{ => pre_merge}/models/__init__.py (95%) create mode 100644 tests/pre_merge/models/test_model.py rename tests/{ => pre_merge}/utils/__init__.py (100%) rename tests/{ => pre_merge}/utils/test_download_progress_bar.py (100%) diff --git a/.github/workflows/tox.yml b/.github/workflows/pre_merge.yml similarity index 100% rename from .github/workflows/tox.yml rename to .github/workflows/pre_merge.yml diff --git a/anomalib/core/callbacks/model_loader.py b/anomalib/core/callbacks/model_loader.py index c74663ddc1..f13ebde541 100644 --- a/anomalib/core/callbacks/model_loader.py +++ b/anomalib/core/callbacks/model_loader.py @@ -14,4 +14,4 @@ def on_test_start(self, trainer, pl_module: LightningModule) -> None: # pylint: Loads the model weights from ``weights_path`` into the PyTorch module. """ - pl_module.load_state_dict(torch.load(self.weights_path)["state_dict"]) + pl_module.load_state_dict(torch.load(self.weights_path, map_location="cpu")["state_dict"]) diff --git a/anomalib/core/metrics/auroc.py b/anomalib/core/metrics/auroc.py index 35165a9090..ffbc142d9b 100644 --- a/anomalib/core/metrics/auroc.py +++ b/anomalib/core/metrics/auroc.py @@ -14,4 +14,4 @@ def compute(self) -> Tensor: Value of the AUROC metric """ fpr, tpr, _thresholds = super().compute() - return auc(fpr, tpr) + return auc(fpr, tpr, reorder=True) diff --git a/anomalib/utils/hpo/__init__.py b/anomalib/utils/hpo/__init__.py new file mode 100644 index 0000000000..df0b042da6 --- /dev/null +++ b/anomalib/utils/hpo/__init__.py @@ -0,0 +1,15 @@ +"""Utils to help in HPO search.""" + +# Copyright (C) 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. diff --git a/anomalib/utils/hpo/config.py b/anomalib/utils/hpo/config.py new file mode 100644 index 0000000000..9edcc944e5 --- /dev/null +++ b/anomalib/utils/hpo/config.py @@ -0,0 +1,53 @@ +"""Utils to update configuration files.""" + +# Copyright (C) 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +from typing import List + +from omegaconf import DictConfig + + +def flatten_sweep_params(params_dict: DictConfig) -> DictConfig: + """Flatten the nested parameters section of the config object. + + Args: + params_dict: DictConfig: The dictionary containing the hpo parameters in the original, nested, structure. + + Returns: + flattened version of the parameter dictionary. + """ + + def process_params(nested_params: DictConfig, keys: List[str], flattened_params: DictConfig): + """Flatten nested dictionary. + + Recursive helper function that traverses the nested config object and stores the leaf nodes in a flattened + dictionary. + + Args: + nested_params: DictConfig: config object containing the original parameters. + keys: List[str]: list of keys leading to the current location in the config. + flattened_params: DictConfig: Dictionary in which the flattened parameters are stored. + """ + for name, cfg in nested_params.items(): + if isinstance(cfg, DictConfig): + process_params(cfg, keys + [str(name)], flattened_params) + else: + key = ".".join(keys + [str(name)]) + flattened_params[key] = cfg + + flattened_params_dict = DictConfig({}) + process_params(params_dict, [], flattened_params_dict) + + return flattened_params_dict diff --git a/tests/helpers/model.py b/tests/helpers/model.py new file mode 100644 index 0000000000..da4b52b5b4 --- /dev/null +++ b/tests/helpers/model.py @@ -0,0 +1,126 @@ +"""Common helpers for both nightly and pre-merge model tests.""" + +# Copyright (C) 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +from typing import Dict, Tuple, Union + +import numpy as np +from omegaconf import DictConfig, ListConfig +from pytorch_lightning import LightningDataModule, Trainer + +from anomalib.config import get_configurable_parameters, update_nncf_config +from anomalib.core.callbacks import get_callbacks +from anomalib.core.callbacks.visualizer_callback import VisualizerCallback +from anomalib.core.model.anomaly_module import AnomalyModule +from anomalib.data import get_datamodule +from anomalib.models import get_model + + +def setup( + model_name: str, + dataset_path: str, + project_path: str, + nncf: bool, + category: str, + score_type: str = None, + weight_file: str = "weights/model.ckpt", + fast_run: bool = False, +) -> Tuple[AnomalyModule, Union[DictConfig, ListConfig], LightningDataModule, Trainer]: + """Train the model based on the parameters passed. + + Args: + model_name (str): Name of the model to train. + dataset_path (str): Location of the dataset. + project_path (str): Path to temporary project folder. + nncf (bool): Add nncf callback. + category (str): Category to train on. + score_type (str, optional): Only used for DFM. Defaults to None. + weight_file (str, optional): Path to weight file. + fast_run (bool, optional): If set to true, the model trains for only 1 epoch. We train for one epoch as + this ensures that both anomalous and non-anomalous images are present in the validation step. + + Returns: + Tuple[AnomalyModule, DictConfig, LightningDataModule, Trainer]: trained model, updated config, datamodule, trainer object + """ + config = get_configurable_parameters(model_name=model_name) + if score_type is not None: + config.model.score_type = score_type + config.project.seed = 1234 + config.dataset.category = category + config.dataset.path = dataset_path + config.model.weight_file = weight_file if weight_file != "" else None # add model weights to the config + + if nncf: + config.optimization.nncf.apply = True + config = update_nncf_config(config) + config.init_weights = None + + # reassign project path as config is updated in `update_config_for_nncf` + config.project.path = project_path + + datamodule = get_datamodule(config) + model = get_model(config) + + callbacks = get_callbacks(config) + + # Force saving the weights after 1 epoch of training. This is used for testing model loading on pre-merge. + if "early_stopping" in config.model.keys() and fast_run == True: + config.model.early_stopping.metric = None + callbacks = get_callbacks(config) + + for index, callback in enumerate(callbacks): + if isinstance(callback, VisualizerCallback): + callbacks.pop(index) + break + + # Train the model. + if fast_run: + config.trainer.max_epochs = 1 + + trainer = Trainer(callbacks=callbacks, **config.trainer) + trainer.fit(model=model, datamodule=datamodule) + return model, config, datamodule, trainer + + +def model_load_test(config: Union[DictConfig, ListConfig], datamodule: LightningDataModule, results: Dict): + """Create a new model based on the weights specified in config. + + Args: + config ([Union[DictConfig, ListConfig]): Model config. + datamodule (LightningDataModule): Dataloader + results (Dict): Results from original model. + + """ + loaded_model = get_model(config) # get new model + + callbacks = get_callbacks(config) + + for index, callback in enumerate(callbacks): + # Remove visualizer callback as saving results takes time + if isinstance(callback, VisualizerCallback): + callbacks.pop(index) + break + + # create new trainer object with LoadModel callback (assumes it is present) + trainer = Trainer(callbacks=callbacks, **config.trainer) + # Assumes the new model has LoadModel callback and the old one had ModelCheckpoint callback + new_results = trainer.test(model=loaded_model, datamodule=datamodule)[0] + assert np.isclose( + results["image_AUROC"], new_results["image_AUROC"] + ), "Loaded model does not yield close performance results" + if config.dataset.task == "segmentation": + assert np.isclose( + results["pixel_AUROC"], new_results["pixel_AUROC"] + ), "Loaded model does not yield close performance results" diff --git a/tests/models/test_model.py b/tests/models/test_model.py deleted file mode 100644 index e20abe3eb3..0000000000 --- a/tests/models/test_model.py +++ /dev/null @@ -1,191 +0,0 @@ -"""Test Models.""" - -# Copyright (C) 2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. - -import os -import random -import tempfile -from functools import wraps - -import numpy as np -import pytest -from pytorch_lightning import Trainer - -from anomalib.config import get_configurable_parameters, update_nncf_config -from anomalib.core.callbacks import get_callbacks -from anomalib.core.callbacks.visualizer_callback import VisualizerCallback -from anomalib.data import get_datamodule -from anomalib.models import get_model -from tests.helpers.dataset import TestDataset, get_dataset_path - - -@pytest.fixture(autouse=True) -def nightly_build() -> bool: - """If NIGHTLY_BUILD env variable is set to TRUE, the tests will use mvtec and check performance.""" - if os.environ["NIGHTLY_BUILD"] == "TRUE": - return True - return False - - -@pytest.fixture(autouse=True) -def category() -> str: - """PyTest fixture to randomly return an MVTec category. - - Returns: - str: Random MVTec category to train/test. - """ - categories = [ - "bottle", - "cable", - "capsule", - "carpet", - "grid", - "hazelnut", - "leather", - "metal_nut", - "pill", - "screw", - "tile", - "toothbrush", - "transistor", - "wood", - "zipper", - ] - - category = random.choice(categories) # nosec - return category - - -class AddDFMScores: - """Function wrapper for checking both scores of DFM.""" - - def __call__(self, func): - @wraps(func) - def inner(*args, **kwds): - if kwds["model_name"] == "dfm": - for score in ["fre", "nll"]: - func(*args, score_type=score, **kwds) - else: - func(*args, **kwds) - - return inner - - -class TestModel: - """Test model.""" - - def _setup(self, model_name, use_mvtec, dataset_path, project_path, nncf, category, score_type=None): - config = get_configurable_parameters(model_name=model_name) - if score_type is not None: - config.model.score_type = score_type - config.project.seed = 1234 - config.dataset.category = category - config.dataset.path = dataset_path - config.model.weight_file = "weights/model.ckpt" # add model weights to the config - - if not use_mvtec: - config.dataset.category = "shapes" - - if nncf: - config.optimization.nncf.apply = True - config = update_nncf_config(config) - config.init_weights = None - - # reassign project path as config is updated in `update_config_for_nncf` - config.project.path = project_path - - datamodule = get_datamodule(config) - model = get_model(config) - - callbacks = get_callbacks(config) - - for index, callback in enumerate(callbacks): - if isinstance(callback, VisualizerCallback): - callbacks.pop(index) - break - - # Train the model. - trainer = Trainer(callbacks=callbacks, **config.trainer) - trainer.fit(model=model, datamodule=datamodule) - return model, config, datamodule, trainer - - def _test_metrics(self, trainer, config, model, datamodule): - """Tests the model metrics but also acts as a setup.""" - - results = trainer.test(model=model, datamodule=datamodule)[0] - - if nightly_build: - assert results["image_AUROC"] >= 0.6 - - if config.dataset.task == "segmentation": - assert results["pixel_AUROC"] >= 0.6 - return results - - def _test_model_load(self, config, datamodule, results): - loaded_model = get_model(config) # get new model - - callbacks = get_callbacks(config) - - for index, callback in enumerate(callbacks): - # Remove visualizer callback as saving results takes time - if isinstance(callback, VisualizerCallback): - callbacks.pop(index) - break - - # create new trainer object with LoadModel callback (assumes it is present) - trainer = Trainer(callbacks=callbacks, **config.trainer) - # Assumes the new model has LoadModel callback and the old one had ModelCheckpoint callback - new_results = trainer.test(model=loaded_model, datamodule=datamodule)[0] - assert np.isclose( - results["image_AUROC"], new_results["image_AUROC"] - ), "Loaded model does not yield close performance results" - if config.dataset.task == "segmentation": - assert np.isclose( - results["pixel_AUROC"], new_results["pixel_AUROC"] - ), "Loaded model does not yield close performance results" - - @pytest.mark.parametrize( - ["model_name", "nncf"], - [ - ("padim", False), - ("dfkde", False), - # ("dfm", False), # skip dfm test - ("stfpm", False), - ("stfpm", True), - ("patchcore", False), - ("cflow", False), - ], - ) - @pytest.mark.flaky(max_runs=3) - @TestDataset(num_train=200, num_test=10, path=get_dataset_path(), use_mvtec=nightly_build) - @AddDFMScores() - def test_model(self, category, model_name, nncf, use_mvtec=nightly_build, path="./datasets/MVTec", score_type=None): - """Driver for all the tests in the class.""" - with tempfile.TemporaryDirectory() as project_path: - model, config, datamodule, trainer = self._setup( - model_name=model_name, - use_mvtec=use_mvtec, - dataset_path=path, - nncf=nncf, - project_path=project_path, - category=category, - score_type=score_type, - ) - - # test model metrics - results = self._test_metrics(trainer=trainer, config=config, model=model, datamodule=datamodule) - - # test model load - self._test_model_load(config=config, datamodule=datamodule, results=results) diff --git a/tests/core/callbacks/compress_callback/__init__.py b/tests/nightly/deploy/__init__.py similarity index 100% rename from tests/core/callbacks/compress_callback/__init__.py rename to tests/nightly/deploy/__init__.py diff --git a/tests/deploy/test_inferencer.py b/tests/nightly/deploy/test_inferencer.py similarity index 97% rename from tests/deploy/test_inferencer.py rename to tests/nightly/deploy/test_inferencer.py index 883f6f49d1..9353aa3094 100644 --- a/tests/deploy/test_inferencer.py +++ b/tests/nightly/deploy/test_inferencer.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions # and limitations under the License. +import os from pathlib import Path from tempfile import TemporaryDirectory from typing import Union @@ -43,6 +44,9 @@ def get_model_config( return model_config +@pytest.mark.skipif( + os.environ["NIGHTLY_BUILD"] == "FALSE", reason="Skipping the test as it is not running nightly build." +) class TestInferencers: @pytest.mark.parametrize( "model_name", diff --git a/tests/core/callbacks/normalization_callback/__init__.py b/tests/nightly/models/__init__.py similarity index 100% rename from tests/core/callbacks/normalization_callback/__init__.py rename to tests/nightly/models/__init__.py diff --git a/tests/nightly/models/test_model.py b/tests/nightly/models/test_model.py new file mode 100644 index 0000000000..4a633cedb4 --- /dev/null +++ b/tests/nightly/models/test_model.py @@ -0,0 +1,130 @@ +"""Test Models on all MVTec Categories.""" + +# Copyright (C) 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +import itertools +import os +import tempfile +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Union + +import numpy as np +import pandas as pd +import pytest +from omegaconf import DictConfig, ListConfig +from pytorch_lightning import Trainer + +from anomalib.core.callbacks import get_callbacks +from anomalib.core.callbacks.visualizer_callback import VisualizerCallback +from anomalib.models import get_model +from anomalib.utils.hpo.config import flatten_sweep_params +from tests.helpers.dataset import get_dataset_path +from tests.helpers.model import model_load_test, setup + + +def get_model_nncf_cat() -> List: + model_support = [ + ("padim", False), + ("dfkde", False), + ("dfm", False), + ("stfpm", False), + ("stfpm", True), + ("patchcore", False), + ("cflow", False), + ] + categories = [ + "bottle", + "cable", + "capsule", + "carpet", + "grid", + "hazelnut", + "leather", + "metal_nut", + "pill", + "screw", + "tile", + "toothbrush", + "transistor", + "wood", + "zipper", + ] + + return [ + (model, nncf, category) for ((model, nncf), category) in list(itertools.product(*[model_support, categories])) + ] + + +@pytest.mark.skipif( + os.environ["NIGHTLY_BUILD"] == "FALSE", reason="Skipping the test as it is not running nightly build." +) +class TestModel: + """Run Model on all categories.""" + + def _test_metrics(self, trainer, config, model, datamodule): + """Tests the model metrics but also acts as a setup.""" + + results = trainer.test(model=model, datamodule=datamodule)[0] + + assert results["image_AUROC"] >= 0.6 + + if config.dataset.task == "segmentation": + assert results["pixel_AUROC"] >= 0.6 + return results + + def _save_to_csv(self, config: Union[DictConfig, ListConfig], results: Dict): + """Save model results to csv. Useful for tracking model drift. + + Args: + config (Union[DictConfig, ListConfig]): Model config which is also added to csv for complete picture. + results (Dict): Metrics from trainer.test + """ + # Save results in csv for tracking model drift + model_metrics = flatten_sweep_params(config) + # convert dict, list values to string + for key, val in model_metrics.items(): + if isinstance(val, (list, dict, ListConfig, DictConfig)): + model_metrics[key] = str(val) + for metric, value in results.items(): + model_metrics[metric] = value + model_metrics_df = pd.DataFrame([model_metrics]) + + result_path = Path(f"tests/artifacts/{datetime.now().strftime('%m_%d_%Y')}.csv") + result_path.parent.mkdir(parents=True, exist_ok=True) + if not result_path.is_file(): + model_metrics_df.to_csv(result_path) + else: + model_metrics_df.to_csv(result_path, mode="a", header=False) + + @pytest.mark.parametrize(["model_name", "nncf", "category"], get_model_nncf_cat()) + def test_model(self, model_name, nncf, category, path=get_dataset_path(), score_type=None): + with tempfile.TemporaryDirectory() as project_path: + model, config, datamodule, trainer = setup( + model_name=model_name, + dataset_path=path, + nncf=nncf, + project_path=project_path, + category=category, + score_type=score_type, + ) + + # test model metrics + results = self._test_metrics(trainer=trainer, config=config, model=model, datamodule=datamodule) + + # test model load + model_load_test(config=config, datamodule=datamodule, results=results) + + self._save_to_csv(config, results) diff --git a/tests/config/__init__.py b/tests/pre_merge/config/__init__.py similarity index 100% rename from tests/config/__init__.py rename to tests/pre_merge/config/__init__.py diff --git a/tests/config/test_config.py b/tests/pre_merge/config/test_config.py similarity index 100% rename from tests/config/test_config.py rename to tests/pre_merge/config/test_config.py diff --git a/tests/core/callbacks/visualizer_callback/__init__.py b/tests/pre_merge/core/callbacks/compress_callback/__init__.py similarity index 100% rename from tests/core/callbacks/visualizer_callback/__init__.py rename to tests/pre_merge/core/callbacks/compress_callback/__init__.py diff --git a/tests/core/callbacks/compress_callback/dummy_config.yml b/tests/pre_merge/core/callbacks/compress_callback/dummy_config.yml similarity index 100% rename from tests/core/callbacks/compress_callback/dummy_config.yml rename to tests/pre_merge/core/callbacks/compress_callback/dummy_config.yml diff --git a/tests/core/callbacks/compress_callback/dummy_lightning_model.py b/tests/pre_merge/core/callbacks/compress_callback/dummy_lightning_model.py similarity index 100% rename from tests/core/callbacks/compress_callback/dummy_lightning_model.py rename to tests/pre_merge/core/callbacks/compress_callback/dummy_lightning_model.py diff --git a/tests/core/callbacks/compress_callback/test_compress.py b/tests/pre_merge/core/callbacks/compress_callback/test_compress.py similarity index 84% rename from tests/core/callbacks/compress_callback/test_compress.py rename to tests/pre_merge/core/callbacks/compress_callback/test_compress.py index 238cd08255..8c50707ad3 100644 --- a/tests/core/callbacks/compress_callback/test_compress.py +++ b/tests/pre_merge/core/callbacks/compress_callback/test_compress.py @@ -6,7 +6,7 @@ from anomalib.config import get_configurable_parameters from anomalib.core.callbacks.compress import CompressModelCallback -from tests.core.callbacks.compress_callback.dummy_lightning_model import ( +from tests.pre_merge.core.callbacks.compress_callback.dummy_lightning_model import ( DummyLightningModule, FakeDataModule, ) @@ -15,7 +15,9 @@ def test_compress_model_callback(): """Tests if an optimized model is created.""" - config = get_configurable_parameters(model_config_path="tests/core/callbacks/compress_callback/dummy_config.yml") + config = get_configurable_parameters( + model_config_path="tests/pre_merge/core/callbacks/compress_callback/dummy_config.yml" + ) with tempfile.TemporaryDirectory() as tmp_dir: config.project.path = tmp_dir diff --git a/tests/datasets/__init__.py b/tests/pre_merge/core/callbacks/normalization_callback/__init__.py similarity index 100% rename from tests/datasets/__init__.py rename to tests/pre_merge/core/callbacks/normalization_callback/__init__.py diff --git a/tests/core/callbacks/normalization_callback/test_normalization_callback.py b/tests/pre_merge/core/callbacks/normalization_callback/test_normalization_callback.py similarity index 100% rename from tests/core/callbacks/normalization_callback/test_normalization_callback.py rename to tests/pre_merge/core/callbacks/normalization_callback/test_normalization_callback.py diff --git a/tests/deploy/__init__.py b/tests/pre_merge/core/callbacks/visualizer_callback/__init__.py similarity index 100% rename from tests/deploy/__init__.py rename to tests/pre_merge/core/callbacks/visualizer_callback/__init__.py diff --git a/tests/core/callbacks/visualizer_callback/dummy_lightning_model.py b/tests/pre_merge/core/callbacks/visualizer_callback/dummy_lightning_model.py similarity index 100% rename from tests/core/callbacks/visualizer_callback/dummy_lightning_model.py rename to tests/pre_merge/core/callbacks/visualizer_callback/dummy_lightning_model.py diff --git a/tests/core/callbacks/visualizer_callback/test_visualizer.py b/tests/pre_merge/core/callbacks/visualizer_callback/test_visualizer.py similarity index 100% rename from tests/core/callbacks/visualizer_callback/test_visualizer.py rename to tests/pre_merge/core/callbacks/visualizer_callback/test_visualizer.py diff --git a/tests/pre_merge/datasets/__init__.py b/tests/pre_merge/datasets/__init__.py new file mode 100644 index 0000000000..1d3fed93f9 --- /dev/null +++ b/tests/pre_merge/datasets/__init__.py @@ -0,0 +1,15 @@ +"""Test dataset, tiler and transforms.""" + +# Copyright (C) 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. diff --git a/tests/datasets/dummy_config.yml b/tests/pre_merge/datasets/dummy_config.yml similarity index 100% rename from tests/datasets/dummy_config.yml rename to tests/pre_merge/datasets/dummy_config.yml diff --git a/tests/datasets/test_dataset.py b/tests/pre_merge/datasets/test_dataset.py similarity index 100% rename from tests/datasets/test_dataset.py rename to tests/pre_merge/datasets/test_dataset.py diff --git a/tests/datasets/test_tiler.py b/tests/pre_merge/datasets/test_tiler.py similarity index 100% rename from tests/datasets/test_tiler.py rename to tests/pre_merge/datasets/test_tiler.py diff --git a/tests/datasets/test_transforms.py b/tests/pre_merge/datasets/test_transforms.py similarity index 100% rename from tests/datasets/test_transforms.py rename to tests/pre_merge/datasets/test_transforms.py diff --git a/tests/loggers/__init__.py b/tests/pre_merge/loggers/__init__.py similarity index 100% rename from tests/loggers/__init__.py rename to tests/pre_merge/loggers/__init__.py diff --git a/tests/loggers/test_get_logger.py b/tests/pre_merge/loggers/test_get_logger.py similarity index 100% rename from tests/loggers/test_get_logger.py rename to tests/pre_merge/loggers/test_get_logger.py diff --git a/tests/models/__init__.py b/tests/pre_merge/models/__init__.py similarity index 95% rename from tests/models/__init__.py rename to tests/pre_merge/models/__init__.py index 28c3d19488..2f80c33af4 100644 --- a/tests/models/__init__.py +++ b/tests/pre_merge/models/__init__.py @@ -1,4 +1,4 @@ -"""Test models.""" +"""Pre-merge model tests.""" # Copyright (C) 2020 Intel Corporation # diff --git a/tests/pre_merge/models/test_model.py b/tests/pre_merge/models/test_model.py new file mode 100644 index 0000000000..318c7878eb --- /dev/null +++ b/tests/pre_merge/models/test_model.py @@ -0,0 +1,67 @@ +"""Quick sanity check on models.""" + +# Copyright (C) 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +import os +import tempfile + +import numpy as np +import pytest +from pytorch_lightning import Trainer + +from anomalib.config import get_configurable_parameters, update_nncf_config +from anomalib.core.callbacks import get_callbacks +from anomalib.core.callbacks.visualizer_callback import VisualizerCallback +from anomalib.data import get_datamodule +from anomalib.models import get_model +from tests.helpers.dataset import TestDataset +from tests.helpers.model import model_load_test, setup + + +@pytest.mark.skipif(os.environ["NIGHTLY_BUILD"] == "TRUE", reason="Skipping the test as it is running nightly build.") +class TestModel: + """Do a sanity check on the models.""" + + @pytest.mark.parametrize( + ["model_name", "nncf"], + [ + ("padim", False), + ("dfkde", False), + ("dfm", False), + ("stfpm", False), + ("stfpm", True), + ("patchcore", False), + ("cflow", False), + ], + ) + @TestDataset(num_train=20, num_test=10) + def test_model(self, model_name, nncf, category="shapes", path=""): + """Test the models on only 1 epoch as a sanity check before merge.""" + with tempfile.TemporaryDirectory() as project_path: + # Train test + model, config, datamodule, trainer = setup( + model_name, + dataset_path=path, + project_path=project_path, + nncf=nncf, + category=category, + weight_file="", + fast_run=True, + ) + results = trainer.test(model=model, datamodule=datamodule)[0] + + # Test model load + config.model.weight_file = "weights/model.ckpt" # add model weights to the config + model_load_test(config, datamodule, results) diff --git a/tests/utils/__init__.py b/tests/pre_merge/utils/__init__.py similarity index 100% rename from tests/utils/__init__.py rename to tests/pre_merge/utils/__init__.py diff --git a/tests/utils/test_download_progress_bar.py b/tests/pre_merge/utils/test_download_progress_bar.py similarity index 100% rename from tests/utils/test_download_progress_bar.py rename to tests/pre_merge/utils/test_download_progress_bar.py From f11e7cc790328397dc60090c3f15b86550ed2dae Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Mon, 24 Jan 2022 15:14:01 +0100 Subject: [PATCH 04/14] =?UTF-8?q?=F0=9F=A9=B9=20Fix=20file=20name=20issue?= =?UTF-8?q?=20for=20pytest=20and=20add=20init?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{test_model.py => test_model_nightly.py} | 0 tests/pre_merge/__init__.py | 15 +++++++++++++++ .../{test_model.py => test_model_premerge.py} | 7 ------- 3 files changed, 15 insertions(+), 7 deletions(-) rename tests/nightly/models/{test_model.py => test_model_nightly.py} (100%) create mode 100644 tests/pre_merge/__init__.py rename tests/pre_merge/models/{test_model.py => test_model_premerge.py} (86%) diff --git a/tests/nightly/models/test_model.py b/tests/nightly/models/test_model_nightly.py similarity index 100% rename from tests/nightly/models/test_model.py rename to tests/nightly/models/test_model_nightly.py diff --git a/tests/pre_merge/__init__.py b/tests/pre_merge/__init__.py new file mode 100644 index 0000000000..da49853df6 --- /dev/null +++ b/tests/pre_merge/__init__.py @@ -0,0 +1,15 @@ +"""Pre-merge tests.""" + +# Copyright (C) 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. diff --git a/tests/pre_merge/models/test_model.py b/tests/pre_merge/models/test_model_premerge.py similarity index 86% rename from tests/pre_merge/models/test_model.py rename to tests/pre_merge/models/test_model_premerge.py index 318c7878eb..c7f045ddef 100644 --- a/tests/pre_merge/models/test_model.py +++ b/tests/pre_merge/models/test_model_premerge.py @@ -17,15 +17,8 @@ import os import tempfile -import numpy as np import pytest -from pytorch_lightning import Trainer -from anomalib.config import get_configurable_parameters, update_nncf_config -from anomalib.core.callbacks import get_callbacks -from anomalib.core.callbacks.visualizer_callback import VisualizerCallback -from anomalib.data import get_datamodule -from anomalib.models import get_model from tests.helpers.dataset import TestDataset from tests.helpers.model import model_load_test, setup From 8f9b6b8d3721a942b9794b3cf418a0173e579348 Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Mon, 24 Jan 2022 16:32:41 +0100 Subject: [PATCH 05/14] =?UTF-8?q?=F0=9F=9A=9A=20Move=20core=20tests=20to?= =?UTF-8?q?=20nightly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/helpers/model.py | 7 ++++++- .../core/callbacks/compress_callback/__init__.py | 0 .../core/callbacks/compress_callback/dummy_config.yml | 0 .../callbacks/compress_callback/dummy_lightning_model.py | 0 .../core/callbacks/compress_callback/test_compress.py | 4 ++++ .../core/callbacks/normalization_callback/__init__.py | 0 .../normalization_callback/test_normalization_callback.py | 6 ++++++ .../core/callbacks/visualizer_callback/__init__.py | 0 .../callbacks/visualizer_callback/dummy_lightning_model.py | 0 .../core/callbacks/visualizer_callback/test_visualizer.py | 3 +++ 10 files changed, 19 insertions(+), 1 deletion(-) rename tests/{pre_merge => nightly}/core/callbacks/compress_callback/__init__.py (100%) rename tests/{pre_merge => nightly}/core/callbacks/compress_callback/dummy_config.yml (100%) rename tests/{pre_merge => nightly}/core/callbacks/compress_callback/dummy_lightning_model.py (100%) rename tests/{pre_merge => nightly}/core/callbacks/compress_callback/test_compress.py (90%) rename tests/{pre_merge => nightly}/core/callbacks/normalization_callback/__init__.py (100%) rename tests/{pre_merge => nightly}/core/callbacks/normalization_callback/test_normalization_callback.py (91%) rename tests/{pre_merge => nightly}/core/callbacks/visualizer_callback/__init__.py (100%) rename tests/{pre_merge => nightly}/core/callbacks/visualizer_callback/dummy_lightning_model.py (100%) rename tests/{pre_merge => nightly}/core/callbacks/visualizer_callback/test_visualizer.py (92%) diff --git a/tests/helpers/model.py b/tests/helpers/model.py index da4b52b5b4..4de5a3f94f 100644 --- a/tests/helpers/model.py +++ b/tests/helpers/model.py @@ -60,7 +60,12 @@ def setup( config.project.seed = 1234 config.dataset.category = category config.dataset.path = dataset_path - config.model.weight_file = weight_file if weight_file != "" else None # add model weights to the config + + # If weight file is empty, remove the key from config + if "weight_file" in config.model.keys() and weight_file == "": + config.model.pop("weight_file") + else: + config.model.weight_file = weight_file if nncf: config.optimization.nncf.apply = True diff --git a/tests/pre_merge/core/callbacks/compress_callback/__init__.py b/tests/nightly/core/callbacks/compress_callback/__init__.py similarity index 100% rename from tests/pre_merge/core/callbacks/compress_callback/__init__.py rename to tests/nightly/core/callbacks/compress_callback/__init__.py diff --git a/tests/pre_merge/core/callbacks/compress_callback/dummy_config.yml b/tests/nightly/core/callbacks/compress_callback/dummy_config.yml similarity index 100% rename from tests/pre_merge/core/callbacks/compress_callback/dummy_config.yml rename to tests/nightly/core/callbacks/compress_callback/dummy_config.yml diff --git a/tests/pre_merge/core/callbacks/compress_callback/dummy_lightning_model.py b/tests/nightly/core/callbacks/compress_callback/dummy_lightning_model.py similarity index 100% rename from tests/pre_merge/core/callbacks/compress_callback/dummy_lightning_model.py rename to tests/nightly/core/callbacks/compress_callback/dummy_lightning_model.py diff --git a/tests/pre_merge/core/callbacks/compress_callback/test_compress.py b/tests/nightly/core/callbacks/compress_callback/test_compress.py similarity index 90% rename from tests/pre_merge/core/callbacks/compress_callback/test_compress.py rename to tests/nightly/core/callbacks/compress_callback/test_compress.py index 8c50707ad3..3126d72961 100644 --- a/tests/pre_merge/core/callbacks/compress_callback/test_compress.py +++ b/tests/nightly/core/callbacks/compress_callback/test_compress.py @@ -1,6 +1,7 @@ import os import tempfile +import pytest import pytorch_lightning as pl from pytorch_lightning.callbacks.early_stopping import EarlyStopping @@ -12,6 +13,9 @@ ) +@pytest.mark.skipif( + os.environ["NIGHTLY_BUILD"] == "FALSE", reason="Skipping the test as it is not running nightly build." +) def test_compress_model_callback(): """Tests if an optimized model is created.""" diff --git a/tests/pre_merge/core/callbacks/normalization_callback/__init__.py b/tests/nightly/core/callbacks/normalization_callback/__init__.py similarity index 100% rename from tests/pre_merge/core/callbacks/normalization_callback/__init__.py rename to tests/nightly/core/callbacks/normalization_callback/__init__.py diff --git a/tests/pre_merge/core/callbacks/normalization_callback/test_normalization_callback.py b/tests/nightly/core/callbacks/normalization_callback/test_normalization_callback.py similarity index 91% rename from tests/pre_merge/core/callbacks/normalization_callback/test_normalization_callback.py rename to tests/nightly/core/callbacks/normalization_callback/test_normalization_callback.py index 14237edfda..7938390569 100644 --- a/tests/pre_merge/core/callbacks/normalization_callback/test_normalization_callback.py +++ b/tests/nightly/core/callbacks/normalization_callback/test_normalization_callback.py @@ -1,3 +1,6 @@ +import os + +import pytest from pytorch_lightning import Trainer, seed_everything from anomalib.config import get_configurable_parameters @@ -17,6 +20,9 @@ def run_train_test(config): return results +@pytest.mark.skipif( + os.environ["NIGHTLY_BUILD"] == "FALSE", reason="Skipping the test as it is not running nightly build." +) def test_normalizer(): config = get_configurable_parameters(model_config_path="anomalib/models/padim/config.yaml") config.dataset.path = get_dataset_path(config.dataset.path) diff --git a/tests/pre_merge/core/callbacks/visualizer_callback/__init__.py b/tests/nightly/core/callbacks/visualizer_callback/__init__.py similarity index 100% rename from tests/pre_merge/core/callbacks/visualizer_callback/__init__.py rename to tests/nightly/core/callbacks/visualizer_callback/__init__.py diff --git a/tests/pre_merge/core/callbacks/visualizer_callback/dummy_lightning_model.py b/tests/nightly/core/callbacks/visualizer_callback/dummy_lightning_model.py similarity index 100% rename from tests/pre_merge/core/callbacks/visualizer_callback/dummy_lightning_model.py rename to tests/nightly/core/callbacks/visualizer_callback/dummy_lightning_model.py diff --git a/tests/pre_merge/core/callbacks/visualizer_callback/test_visualizer.py b/tests/nightly/core/callbacks/visualizer_callback/test_visualizer.py similarity index 92% rename from tests/pre_merge/core/callbacks/visualizer_callback/test_visualizer.py rename to tests/nightly/core/callbacks/visualizer_callback/test_visualizer.py index a1f37a57ba..5cad60280b 100644 --- a/tests/pre_merge/core/callbacks/visualizer_callback/test_visualizer.py +++ b/tests/nightly/core/callbacks/visualizer_callback/test_visualizer.py @@ -21,6 +21,9 @@ def get_dummy_logger(config, tempdir): return logger +@pytest.mark.skipif( + os.environ["NIGHTLY_BUILD"] == "FALSE", reason="Skipping the test as it is not running nightly build." +) @pytest.mark.parametrize("dataset", ["segmentation"]) def test_add_images(dataset): """Tests if tensorboard logs are generated.""" From acb441d012f76e68a3a299b9884a05716a19618d Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Mon, 24 Jan 2022 16:44:18 +0100 Subject: [PATCH 06/14] =?UTF-8?q?=F0=9F=94=A8=20fix=20import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/nightly/core/callbacks/compress_callback/test_compress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nightly/core/callbacks/compress_callback/test_compress.py b/tests/nightly/core/callbacks/compress_callback/test_compress.py index 3126d72961..eeaa1ce6a4 100644 --- a/tests/nightly/core/callbacks/compress_callback/test_compress.py +++ b/tests/nightly/core/callbacks/compress_callback/test_compress.py @@ -7,7 +7,7 @@ from anomalib.config import get_configurable_parameters from anomalib.core.callbacks.compress import CompressModelCallback -from tests.pre_merge.core.callbacks.compress_callback.dummy_lightning_model import ( +from tests.nightly.core.callbacks.compress_callback.dummy_lightning_model import ( DummyLightningModule, FakeDataModule, ) From d13b03482d9c78fb53a81fd69a9c1ac131ee1056 Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Mon, 31 Jan 2022 10:29:25 +0100 Subject: [PATCH 07/14] Add nightly to tox and remove os.environ --- .github/workflows/nightly.yml | 3 +- .github/workflows/pre_merge.yml | 3 +- anomalib/core/callbacks/model_loader.py | 2 +- anomalib/core/metrics/auroc.py | 2 +- tests/helpers/model.py | 26 +- .../compress_callback/test_compress.py | 5 +- .../test_normalization_callback.py | 6 - .../visualizer_callback/test_visualizer.py | 3 - tests/nightly/deploy/test_inferencer.py | 3 - .../models/performance_thresholds.yaml | 296 ++++++++++++++++++ tests/nightly/models/test_model_nightly.py | 26 +- tests/pre_merge/models/test_model_premerge.py | 5 +- tox.ini | 28 +- 13 files changed, 357 insertions(+), 51 deletions(-) create mode 100644 tests/nightly/models/performance_thresholds.yaml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 519d3ff691..1813b0c49d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -22,8 +22,7 @@ jobs: run: | export ANOMALIB_DATASET_PATH=/media/data1/datasets/MVTec export CUDA_VISIBLE_DEVICES=2 - export NIGHTLY_BUILD=TRUE - tox -e coverage + tox -e nightly - name: Upload coverage result uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/pre_merge.yml b/.github/workflows/pre_merge.yml index 8f1359c0e1..f6151ca73e 100644 --- a/.github/workflows/pre_merge.yml +++ b/.github/workflows/pre_merge.yml @@ -24,8 +24,7 @@ jobs: run: | export ANOMALIB_DATASET_PATH=/media/data1/datasets/MVTec export CUDA_VISIBLE_DEVICES=3 - export NIGHTLY_BUILD=FALSE - tox -e coverage + tox -e pre_merge - name: Upload coverage result uses: actions/upload-artifact@v2 with: diff --git a/anomalib/core/callbacks/model_loader.py b/anomalib/core/callbacks/model_loader.py index f13ebde541..c74663ddc1 100644 --- a/anomalib/core/callbacks/model_loader.py +++ b/anomalib/core/callbacks/model_loader.py @@ -14,4 +14,4 @@ def on_test_start(self, trainer, pl_module: LightningModule) -> None: # pylint: Loads the model weights from ``weights_path`` into the PyTorch module. """ - pl_module.load_state_dict(torch.load(self.weights_path, map_location="cpu")["state_dict"]) + pl_module.load_state_dict(torch.load(self.weights_path)["state_dict"]) diff --git a/anomalib/core/metrics/auroc.py b/anomalib/core/metrics/auroc.py index ffbc142d9b..35165a9090 100644 --- a/anomalib/core/metrics/auroc.py +++ b/anomalib/core/metrics/auroc.py @@ -14,4 +14,4 @@ def compute(self) -> Tensor: Value of the AUROC metric """ fpr, tpr, _thresholds = super().compute() - return auc(fpr, tpr, reorder=True) + return auc(fpr, tpr) diff --git a/tests/helpers/model.py b/tests/helpers/model.py index 4de5a3f94f..6f63b38006 100644 --- a/tests/helpers/model.py +++ b/tests/helpers/model.py @@ -14,11 +14,13 @@ # See the License for the specific language governing permissions # and limitations under the License. +import os from typing import Dict, Tuple, Union import numpy as np from omegaconf import DictConfig, ListConfig from pytorch_lightning import LightningDataModule, Trainer +from pytorch_lightning.callbacks import ModelCheckpoint from anomalib.config import get_configurable_parameters, update_nncf_config from anomalib.core.callbacks import get_callbacks @@ -37,7 +39,7 @@ def setup( score_type: str = None, weight_file: str = "weights/model.ckpt", fast_run: bool = False, -) -> Tuple[AnomalyModule, Union[DictConfig, ListConfig], LightningDataModule, Trainer]: +) -> Tuple[Union[DictConfig, ListConfig], LightningDataModule, AnomalyModule, Trainer]: """Train the model based on the parameters passed. Args: @@ -52,7 +54,7 @@ def setup( this ensures that both anomalous and non-anomalous images are present in the validation step. Returns: - Tuple[AnomalyModule, DictConfig, LightningDataModule, Trainer]: trained model, updated config, datamodule, trainer object + Tuple[DictConfig, LightningDataModule, AnomalyModule, Trainer]: config, datamodule, trained model, trainer """ config = get_configurable_parameters(model_name=model_name) if score_type is not None: @@ -80,10 +82,20 @@ def setup( callbacks = get_callbacks(config) - # Force saving the weights after 1 epoch of training. This is used for testing model loading on pre-merge. - if "early_stopping" in config.model.keys() and fast_run == True: - config.model.early_stopping.metric = None - callbacks = get_callbacks(config) + # Force model checkpoint to create checkpoint after first epoch + if fast_run == True: + for index, callback in enumerate(callbacks): + if isinstance(callback, ModelCheckpoint): + callbacks.pop(index) + model_checkpoint = ModelCheckpoint( + dirpath=os.path.join(config.project.path, "weights"), + filename="model", + monitor=None, + mode="max", + auto_insert_metric_name=False, + ) + callbacks.append(model_checkpoint) + break for index, callback in enumerate(callbacks): if isinstance(callback, VisualizerCallback): @@ -96,7 +108,7 @@ def setup( trainer = Trainer(callbacks=callbacks, **config.trainer) trainer.fit(model=model, datamodule=datamodule) - return model, config, datamodule, trainer + return config, datamodule, model, trainer def model_load_test(config: Union[DictConfig, ListConfig], datamodule: LightningDataModule, results: Dict): diff --git a/tests/nightly/core/callbacks/compress_callback/test_compress.py b/tests/nightly/core/callbacks/compress_callback/test_compress.py index eeaa1ce6a4..6e65ec6831 100644 --- a/tests/nightly/core/callbacks/compress_callback/test_compress.py +++ b/tests/nightly/core/callbacks/compress_callback/test_compress.py @@ -13,14 +13,11 @@ ) -@pytest.mark.skipif( - os.environ["NIGHTLY_BUILD"] == "FALSE", reason="Skipping the test as it is not running nightly build." -) def test_compress_model_callback(): """Tests if an optimized model is created.""" config = get_configurable_parameters( - model_config_path="tests/pre_merge/core/callbacks/compress_callback/dummy_config.yml" + model_config_path="tests/nightly/core/callbacks/compress_callback/dummy_config.yml" ) with tempfile.TemporaryDirectory() as tmp_dir: diff --git a/tests/nightly/core/callbacks/normalization_callback/test_normalization_callback.py b/tests/nightly/core/callbacks/normalization_callback/test_normalization_callback.py index 7938390569..14237edfda 100644 --- a/tests/nightly/core/callbacks/normalization_callback/test_normalization_callback.py +++ b/tests/nightly/core/callbacks/normalization_callback/test_normalization_callback.py @@ -1,6 +1,3 @@ -import os - -import pytest from pytorch_lightning import Trainer, seed_everything from anomalib.config import get_configurable_parameters @@ -20,9 +17,6 @@ def run_train_test(config): return results -@pytest.mark.skipif( - os.environ["NIGHTLY_BUILD"] == "FALSE", reason="Skipping the test as it is not running nightly build." -) def test_normalizer(): config = get_configurable_parameters(model_config_path="anomalib/models/padim/config.yaml") config.dataset.path = get_dataset_path(config.dataset.path) diff --git a/tests/nightly/core/callbacks/visualizer_callback/test_visualizer.py b/tests/nightly/core/callbacks/visualizer_callback/test_visualizer.py index 5cad60280b..a1f37a57ba 100644 --- a/tests/nightly/core/callbacks/visualizer_callback/test_visualizer.py +++ b/tests/nightly/core/callbacks/visualizer_callback/test_visualizer.py @@ -21,9 +21,6 @@ def get_dummy_logger(config, tempdir): return logger -@pytest.mark.skipif( - os.environ["NIGHTLY_BUILD"] == "FALSE", reason="Skipping the test as it is not running nightly build." -) @pytest.mark.parametrize("dataset", ["segmentation"]) def test_add_images(dataset): """Tests if tensorboard logs are generated.""" diff --git a/tests/nightly/deploy/test_inferencer.py b/tests/nightly/deploy/test_inferencer.py index 9353aa3094..23d11ab64e 100644 --- a/tests/nightly/deploy/test_inferencer.py +++ b/tests/nightly/deploy/test_inferencer.py @@ -44,9 +44,6 @@ def get_model_config( return model_config -@pytest.mark.skipif( - os.environ["NIGHTLY_BUILD"] == "FALSE", reason="Skipping the test as it is not running nightly build." -) class TestInferencers: @pytest.mark.parametrize( "model_name", diff --git a/tests/nightly/models/performance_thresholds.yaml b/tests/nightly/models/performance_thresholds.yaml new file mode 100644 index 0000000000..b12e7eea2e --- /dev/null +++ b/tests/nightly/models/performance_thresholds.yaml @@ -0,0 +1,296 @@ +padim: + bottle: + image_AUROC: 0.993 + pixel_AUROC: 0.983 + cable: + image_AUROC: 0.830 + pixel_AUROC: 0.947 + capsule: + image_AUROC: 0.860 + pixel_AUROC: 0.984 + carpet: + image_AUROC: 0.945 + pixel_AUROC: 0.984 + grid: + image_AUROC: 0.857 + pixel_AUROC: 0.941 + hazelnut: + image_AUROC: 0.750 + pixel_AUROC: 0.981 + leather: + image_AUROC: 0.982 + pixel_AUROC: 0.991 + metal_nut: + image_AUROC: 0.968 + pixel_AUROC: 0.971 + pill: + image_AUROC: 0.851 + pixel_AUROC: 0.956 + screw: + image_AUROC: 0.759 + pixel_AUROC: 0.975 + tile: + image_AUROC: 0.943 + pixel_AUROC: 0.933 + toothbrush: + image_AUROC: 0.844 + pixel_AUROC: 0.988 + transistor: + image_AUROC: 0.919 + pixel_AUROC: 0.970 + wood: + image_AUROC: 0.976 + pixel_AUROC: 0.940 + zipper: + image_AUROC: 0.738 + pixel_AUROC: 0.970 + +dfkde: + bottle: + image_AUROC: 0.969 + cable: + image_AUROC: 0.671 + capsule: + image_AUROC: 0.782 + carpet: + image_AUROC: 0.621 + grid: + image_AUROC: 0.494 + hazelnut: + image_AUROC: 0.786 + leather: + image_AUROC: 0.757 + metal_nut: + image_AUROC: 0.759 + pill: + image_AUROC: 0.642 + screw: + image_AUROC: 0.681 + tile: + image_AUROC: 0.965 + toothbrush: + image_AUROC: 0.830 + transistor: + image_AUROC: 0.799 + wood: + image_AUROC: 0.851 + zipper: + image_AUROC: 0.858 + +dfm: + bottle: + image_AUROC: 0.996 + cable: + image_AUROC: 0.926 + capsule: + image_AUROC: 0.876 + carpet: + image_AUROC: 0.846 + grid: + image_AUROC: 0.496 + hazelnut: + image_AUROC: 0.984 + leather: + image_AUROC: 0.931 + metal_nut: + image_AUROC: 0.946 + pill: + image_AUROC: 0.816 + screw: + image_AUROC: 0.760 + tile: + image_AUROC: 0.980 + toothbrush: + image_AUROC: 0.963 + transistor: + image_AUROC: 0.935 + wood: + image_AUROC: 0.957 + zipper: + image_AUROC: 0.945 + +stfpm: + bottle: + image_AUROC: 0.998 + pixel_AUROC: 0.978 + nncf: + image_AUROC: 0.998 + pixel_AUROC: 0.964 + cable: + image_AUROC: 0.939 + pixel_AUROC: 0.947 + nncf: + image_AUROC: 0.937 + pixel_AUROC: 0.936 + capsule: + image_AUROC: 0.668 + pixel_AUROC: 0.961 + nncf: + image_AUROC: 0.657 + pixel_AUROC: 0.967 + carpet: + image_AUROC: 0.986 + pixel_AUROC: 0.987 + nncf: + image_AUROC: 0.991 + pixel_AUROC: 0.987 + grid: + image_AUROC: 0.986 + pixel_AUROC: 0.989 + nncf: + image_AUROC: 0.989 + pixel_AUROC: 0.988 + hazelnut: + image_AUROC: 0.998 + pixel_AUROC: 0.986 + nncf: + image_AUROC: 0.991 + pixel_AUROC: 0.976 + leather: + image_AUROC: 0.995 + pixel_AUROC: 0.983 + nncf: + image_AUROC: 0.988 + pixel_AUROC: 0.974 + metal_nut: + image_AUROC: 0.985 + pixel_AUROC: 0.976 + nncf: + image_AUROC: 0.982 + pixel_AUROC: 0.962 + pill: + image_AUROC: 0.584 + pixel_AUROC: 0.932 + nncf: + image_AUROC: 0.613 + pixel_AUROC: 0.882 + screw: + image_AUROC: 0.766 + pixel_AUROC: 0.973 + nncf: + image_AUROC: 0.452 + pixel_AUROC: 0.952 + tile: + image_AUROC: 0.955 + pixel_AUROC: 0.967 + nncf: + image_AUROC: 0.944 + pixel_AUROC: 0.945 + toothbrush: + image_AUROC: 0.883 + pixel_AUROC: 0.986 + nncf: + image_AUROC: 0.791 + pixel_AUROC: 0.956 + transistor: + image_AUROC: 0.806 + pixel_AUROC: 0.806 + nncf: + image_AUROC: 0.798 + pixel_AUROC: 0.793 + wood: + image_AUROC: 0.989 + pixel_AUROC: 0.948 + nncf: + image_AUROC: 0.992 + pixel_AUROC: 0.951 + zipper: + image_AUROC: 0.837 + pixel_AUROC: 0.982 + nncf: + image_AUROC: 0.843 + pixel_AUROC: 0.980 + +patchcore: + bottle: + image_AUROC: 1.0 + pixel_AUROC: 0.984 + cable: + image_AUROC: 0.994 + pixel_AUROC: 0.987 + capsule: + image_AUROC: 0.980 + pixel_AUROC: 0.987 + carpet: + image_AUROC: 0.980 + pixel_AUROC: 0.988 + grid: + image_AUROC: 0.961 + pixel_AUROC: 0.868 + hazelnut: + image_AUROC: 1.0 + pixel_AUROC: 0.987 + leather: + image_AUROC: 1.0 + pixel_AUROC: 0.990 + metal_nut: + image_AUROC: 0.994 + pixel_AUROC: 0.989 + pill: + image_AUROC: 0.927 + pixel_AUROC: 0.980 + screw: + image_AUROC: 0.928 + pixel_AUROC: 0.989 + tile: + image_AUROC: 1.0 + pixel_AUROC: 0.960 + toothbrush: + image_AUROC: 0.955 + pixel_AUROC: 0.988 + transistor: + image_AUROC: 0.999 + pixel_AUROC: 0.981 + wood: + image_AUROC: 0.989 + pixel_AUROC: 0.934 + zipper: + image_AUROC: 0.981 + pixel_AUROC: 0.983 + +cflow: + bottle: + image_AUROC: 1.0 + pixel_AUROC: 0.980 + cable: + image_AUROC: 0.975 + pixel_AUROC: 0.974 + capsule: + image_AUROC: 0.977 + pixel_AUROC: 0.989 + carpet: + image_AUROC: 0.982 + pixel_AUROC: 0.988 + grid: + image_AUROC: 0.959 + pixel_AUROC: 0.965 + hazelnut: + image_AUROC: 0.999 + pixel_AUROC: 0.986 + leather: + image_AUROC: 1.0 + pixel_AUROC: 0.994 + metal_nut: + image_AUROC: 0.984 + pixel_AUROC: 0.982 + pill: + image_AUROC: 0.967 + pixel_AUROC: 0.986 + screw: + image_AUROC: 0.910 + pixel_AUROC: 0.984 + tile: + image_AUROC: 0.998 + pixel_AUROC: 0.965 + toothbrush: + image_AUROC: 0.9 + pixel_AUROC: 0.985 + transistor: + image_AUROC: 0.9733 + pixel_AUROC: 0.942 + wood: + image_AUROC: 0.991 + pixel_AUROC: 0.937 + zipper: + image_AUROC: 0.991 + pixel_AUROC: 0.982 diff --git a/tests/nightly/models/test_model_nightly.py b/tests/nightly/models/test_model_nightly.py index 4a633cedb4..4cbc13e418 100644 --- a/tests/nightly/models/test_model_nightly.py +++ b/tests/nightly/models/test_model_nightly.py @@ -15,21 +15,16 @@ # and limitations under the License. import itertools -import os import tempfile from datetime import datetime from pathlib import Path from typing import Dict, List, Union -import numpy as np import pandas as pd import pytest -from omegaconf import DictConfig, ListConfig -from pytorch_lightning import Trainer +from omegaconf import DictConfig, ListConfig, OmegaConf +from pytorch_lightning import seed_everything -from anomalib.core.callbacks import get_callbacks -from anomalib.core.callbacks.visualizer_callback import VisualizerCallback -from anomalib.models import get_model from anomalib.utils.hpo.config import flatten_sweep_params from tests.helpers.dataset import get_dataset_path from tests.helpers.model import model_load_test, setup @@ -68,9 +63,6 @@ def get_model_nncf_cat() -> List: ] -@pytest.mark.skipif( - os.environ["NIGHTLY_BUILD"] == "FALSE", reason="Skipping the test as it is not running nightly build." -) class TestModel: """Run Model on all categories.""" @@ -79,10 +71,15 @@ def _test_metrics(self, trainer, config, model, datamodule): results = trainer.test(model=model, datamodule=datamodule)[0] - assert results["image_AUROC"] >= 0.6 + thresholds = OmegaConf.load("tests/nightly/models/performance_thresholds.yaml") + + threshold = thresholds[config.model.name][config.dataset.category] + if "optimization" in config.keys() and config.optimization.nncf.apply: + threshold = threshold.nncf + assert results["image_AUROC"] >= threshold["image_AUROC"] if config.dataset.task == "segmentation": - assert results["pixel_AUROC"] >= 0.6 + assert results["pixel_AUROC"] >= threshold["pixel_AUROC"] return results def _save_to_csv(self, config: Union[DictConfig, ListConfig], results: Dict): @@ -111,8 +108,11 @@ def _save_to_csv(self, config: Union[DictConfig, ListConfig], results: Dict): @pytest.mark.parametrize(["model_name", "nncf", "category"], get_model_nncf_cat()) def test_model(self, model_name, nncf, category, path=get_dataset_path(), score_type=None): + # Fix seed + seed_everything(42) + with tempfile.TemporaryDirectory() as project_path: - model, config, datamodule, trainer = setup( + config, datamodule, model, trainer = setup( model_name=model_name, dataset_path=path, nncf=nncf, diff --git a/tests/pre_merge/models/test_model_premerge.py b/tests/pre_merge/models/test_model_premerge.py index c7f045ddef..2b8a85cdc6 100644 --- a/tests/pre_merge/models/test_model_premerge.py +++ b/tests/pre_merge/models/test_model_premerge.py @@ -23,7 +23,6 @@ from tests.helpers.model import model_load_test, setup -@pytest.mark.skipif(os.environ["NIGHTLY_BUILD"] == "TRUE", reason="Skipping the test as it is running nightly build.") class TestModel: """Do a sanity check on the models.""" @@ -44,17 +43,15 @@ def test_model(self, model_name, nncf, category="shapes", path=""): """Test the models on only 1 epoch as a sanity check before merge.""" with tempfile.TemporaryDirectory() as project_path: # Train test - model, config, datamodule, trainer = setup( + config, datamodule, model, trainer = setup( model_name, dataset_path=path, project_path=project_path, nncf=nncf, category=category, - weight_file="", fast_run=True, ) results = trainer.test(model=model, datamodule=datamodule)[0] # Test model load - config.model.weight_file = "weights/model.ckpt" # add model weights to the config model_load_test(config, datamodule, results) diff --git a/tox.ini b/tox.ini index 6543da9c75..763f799fdd 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,8 @@ envlist = pylint mypy pydocstyle - coverage + pre_merge + nightly [testenv:black] basepython = python3 @@ -57,14 +58,13 @@ deps = pydocstyle commands = pydocstyle anomalib --config=tox.ini -[testenv:coverage] +[testenv:pre_merge] basepython = python3 passenv = ftp_proxy HTTP_PROXY HTTPS_PROXY CUDA_VISIBLE_DEVICES ANOMALIB_DATASET_PATH - NIGHTLY_BUILD deps = coverage pytest @@ -73,10 +73,28 @@ deps = -r{toxinidir}/requirements/openvino.txt commands = coverage erase - coverage run --include=anomalib/* -m pytest -ra - coverage report -m --fail-under=90 + coverage run --include=anomalib/* -m pytest tests/pre_merge/ -ra + coverage report -m --fail-under=85 coverage xml -o {toxworkdir}/coverage.xml +[testenv:nightly] +basepython = python3 +passenv = ftp_proxy + HTTP_PROXY + HTTPS_PROXY + CUDA_VISIBLE_DEVICES + ANOMALIB_DATASET_PATH +deps = + coverage + pytest + flaky + -r{toxinidir}/requirements/base.txt + -r{toxinidir}/requirements/openvino.txt +commands = + coverage erase + coverage run --include=anomalib/* -m pytest tests/nightly/ -ra + coverage report -m --fail-under=90 + coverage xml -o {toxworkdir}/coverage.xml [flake8] max-line-length = 120 From b0a52ec924013f0272125d49c46261edace9cb71 Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Tue, 1 Feb 2022 10:57:18 +0100 Subject: [PATCH 08/14] Update thresholds --- .../models/performance_thresholds.yaml | 22 +++++++++---------- tox.ini | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/nightly/models/performance_thresholds.yaml b/tests/nightly/models/performance_thresholds.yaml index b12e7eea2e..b4ba371be0 100644 --- a/tests/nightly/models/performance_thresholds.yaml +++ b/tests/nightly/models/performance_thresholds.yaml @@ -253,14 +253,14 @@ cflow: image_AUROC: 1.0 pixel_AUROC: 0.980 cable: - image_AUROC: 0.975 - pixel_AUROC: 0.974 + image_AUROC: 0.960 + pixel_AUROC: 0.969 capsule: image_AUROC: 0.977 pixel_AUROC: 0.989 carpet: - image_AUROC: 0.982 - pixel_AUROC: 0.988 + image_AUROC: 0.979 + pixel_AUROC: 0.987 grid: image_AUROC: 0.959 pixel_AUROC: 0.965 @@ -272,9 +272,9 @@ cflow: pixel_AUROC: 0.994 metal_nut: image_AUROC: 0.984 - pixel_AUROC: 0.982 + pixel_AUROC: 0.981 pill: - image_AUROC: 0.967 + image_AUROC: 0.933 pixel_AUROC: 0.986 screw: image_AUROC: 0.910 @@ -283,14 +283,14 @@ cflow: image_AUROC: 0.998 pixel_AUROC: 0.965 toothbrush: - image_AUROC: 0.9 - pixel_AUROC: 0.985 + image_AUROC: 0.894 + pixel_AUROC: 0.984 transistor: image_AUROC: 0.9733 pixel_AUROC: 0.942 wood: - image_AUROC: 0.991 + image_AUROC: 0.942 pixel_AUROC: 0.937 zipper: - image_AUROC: 0.991 - pixel_AUROC: 0.982 + image_AUROC: 0.979 + pixel_AUROC: 0.981 diff --git a/tox.ini b/tox.ini index 763f799fdd..3d737522d6 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ envlist = [testenv:black] basepython = python3 -deps = black +deps = black==20.8b1 commands = black --check --diff anomalib -l 120 [testenv:isort] From 1cac2ed9e2e7757a1413edd33d29002f0c5d515e Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Tue, 1 Feb 2022 11:17:06 +0100 Subject: [PATCH 09/14] Fix isort issues --- anomalib/loggers/wandb.py | 3 ++- tox.ini | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/anomalib/loggers/wandb.py b/anomalib/loggers/wandb.py index 96d4ac79a9..e037e27785 100644 --- a/anomalib/loggers/wandb.py +++ b/anomalib/loggers/wandb.py @@ -17,11 +17,12 @@ from typing import Any, List, Optional, Union import numpy as np -import wandb from matplotlib.figure import Figure from pytorch_lightning.loggers.wandb import WandbLogger from pytorch_lightning.utilities import rank_zero_only +import wandb + from .base import ImageLoggerBase diff --git a/tox.ini b/tox.ini index 3d737522d6..e80d2ac1cf 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ commands = black --check --diff anomalib -l 120 [testenv:isort] basepython = python3 -deps = isort +deps = isort==5.8.0 commands = isort --check-only --df anomalib --profile=black [testenv:flake8] From 65eb87728a06309fff826c2bd10c5a3877715400 Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Tue, 1 Feb 2022 11:53:02 +0100 Subject: [PATCH 10/14] Fix isort issues --- anomalib/models/cflow/README.md | 1 - pyproject.toml | 1 + tox.ini | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/anomalib/models/cflow/README.md b/anomalib/models/cflow/README.md index e213ff7258..3e8fe2e242 100644 --- a/anomalib/models/cflow/README.md +++ b/anomalib/models/cflow/README.md @@ -47,4 +47,3 @@ All results gathered with seed `42`. ![Sample Result 2](../../../docs/source/images/cflow/results/1.png "Sample Result 2") ![Sample Result 3](../../../docs/source/images/cflow/results/2.png "Sample Result 3") - diff --git a/pyproject.toml b/pyproject.toml index dcea1fa272..5b0859faee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ build-backend = "setuptools.build_meta" [tool.isort] profile = "black" +known_first_party = "wandb" sections = ['FUTURE', 'STDLIB', 'THIRDPARTY', 'FIRSTPARTY', 'LOCALFOLDER'] [tool.black] diff --git a/tox.ini b/tox.ini index e80d2ac1cf..5389e4b6c5 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ commands = black --check --diff anomalib -l 120 [testenv:isort] basepython = python3 -deps = isort==5.8.0 +deps = isort==5.10.1 commands = isort --check-only --df anomalib --profile=black [testenv:flake8] From 84a812feb6fc1573b6593ad90f874cd898ef201c Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Wed, 2 Feb 2022 11:30:49 +0100 Subject: [PATCH 11/14] =?UTF-8?q?=F0=9F=9A=9A=20move=20tests/nightly/core?= =?UTF-8?q?=20to=20tests/nightly/utils?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/nightly/models/test_model_nightly.py | 5 +++++ .../{core => utils}/callbacks/compress_callback/__init__.py | 0 .../callbacks/compress_callback/dummy_config.yml | 0 .../callbacks/compress_callback/dummy_lightning_model.py | 0 .../callbacks/compress_callback/test_compress.py | 5 ++--- .../callbacks/normalization_callback/__init__.py | 0 .../normalization_callback/test_normalization_callback.py | 0 .../callbacks/visualizer_callback/__init__.py | 0 .../callbacks/visualizer_callback/dummy_lightning_model.py | 0 .../callbacks/visualizer_callback/test_visualizer.py | 0 10 files changed, 7 insertions(+), 3 deletions(-) rename tests/nightly/{core => utils}/callbacks/compress_callback/__init__.py (100%) rename tests/nightly/{core => utils}/callbacks/compress_callback/dummy_config.yml (100%) rename tests/nightly/{core => utils}/callbacks/compress_callback/dummy_lightning_model.py (100%) rename tests/nightly/{core => utils}/callbacks/compress_callback/test_compress.py (86%) rename tests/nightly/{core => utils}/callbacks/normalization_callback/__init__.py (100%) rename tests/nightly/{core => utils}/callbacks/normalization_callback/test_normalization_callback.py (100%) rename tests/nightly/{core => utils}/callbacks/visualizer_callback/__init__.py (100%) rename tests/nightly/{core => utils}/callbacks/visualizer_callback/dummy_lightning_model.py (100%) rename tests/nightly/{core => utils}/callbacks/visualizer_callback/test_visualizer.py (100%) diff --git a/tests/nightly/models/test_model_nightly.py b/tests/nightly/models/test_model_nightly.py index 4cbc13e418..15ea332a7a 100644 --- a/tests/nightly/models/test_model_nightly.py +++ b/tests/nightly/models/test_model_nightly.py @@ -31,6 +31,11 @@ def get_model_nncf_cat() -> List: + """Test helper for getting cartesian product of models and categories. + + Returns: + List: Returns a combination of models with their nncf support for each category. + """ model_support = [ ("padim", False), ("dfkde", False), diff --git a/tests/nightly/core/callbacks/compress_callback/__init__.py b/tests/nightly/utils/callbacks/compress_callback/__init__.py similarity index 100% rename from tests/nightly/core/callbacks/compress_callback/__init__.py rename to tests/nightly/utils/callbacks/compress_callback/__init__.py diff --git a/tests/nightly/core/callbacks/compress_callback/dummy_config.yml b/tests/nightly/utils/callbacks/compress_callback/dummy_config.yml similarity index 100% rename from tests/nightly/core/callbacks/compress_callback/dummy_config.yml rename to tests/nightly/utils/callbacks/compress_callback/dummy_config.yml diff --git a/tests/nightly/core/callbacks/compress_callback/dummy_lightning_model.py b/tests/nightly/utils/callbacks/compress_callback/dummy_lightning_model.py similarity index 100% rename from tests/nightly/core/callbacks/compress_callback/dummy_lightning_model.py rename to tests/nightly/utils/callbacks/compress_callback/dummy_lightning_model.py diff --git a/tests/nightly/core/callbacks/compress_callback/test_compress.py b/tests/nightly/utils/callbacks/compress_callback/test_compress.py similarity index 86% rename from tests/nightly/core/callbacks/compress_callback/test_compress.py rename to tests/nightly/utils/callbacks/compress_callback/test_compress.py index 6e65ec6831..ffd3a2ee0f 100644 --- a/tests/nightly/core/callbacks/compress_callback/test_compress.py +++ b/tests/nightly/utils/callbacks/compress_callback/test_compress.py @@ -1,13 +1,12 @@ import os import tempfile -import pytest import pytorch_lightning as pl from pytorch_lightning.callbacks.early_stopping import EarlyStopping from anomalib.config import get_configurable_parameters from anomalib.core.callbacks.compress import CompressModelCallback -from tests.nightly.core.callbacks.compress_callback.dummy_lightning_model import ( +from tests.nightly.utils.callbacks.compress_callback.dummy_lightning_model import ( DummyLightningModule, FakeDataModule, ) @@ -17,7 +16,7 @@ def test_compress_model_callback(): """Tests if an optimized model is created.""" config = get_configurable_parameters( - model_config_path="tests/nightly/core/callbacks/compress_callback/dummy_config.yml" + model_config_path="tests/nightly/utils/callbacks/compress_callback/dummy_config.yml" ) with tempfile.TemporaryDirectory() as tmp_dir: diff --git a/tests/nightly/core/callbacks/normalization_callback/__init__.py b/tests/nightly/utils/callbacks/normalization_callback/__init__.py similarity index 100% rename from tests/nightly/core/callbacks/normalization_callback/__init__.py rename to tests/nightly/utils/callbacks/normalization_callback/__init__.py diff --git a/tests/nightly/core/callbacks/normalization_callback/test_normalization_callback.py b/tests/nightly/utils/callbacks/normalization_callback/test_normalization_callback.py similarity index 100% rename from tests/nightly/core/callbacks/normalization_callback/test_normalization_callback.py rename to tests/nightly/utils/callbacks/normalization_callback/test_normalization_callback.py diff --git a/tests/nightly/core/callbacks/visualizer_callback/__init__.py b/tests/nightly/utils/callbacks/visualizer_callback/__init__.py similarity index 100% rename from tests/nightly/core/callbacks/visualizer_callback/__init__.py rename to tests/nightly/utils/callbacks/visualizer_callback/__init__.py diff --git a/tests/nightly/core/callbacks/visualizer_callback/dummy_lightning_model.py b/tests/nightly/utils/callbacks/visualizer_callback/dummy_lightning_model.py similarity index 100% rename from tests/nightly/core/callbacks/visualizer_callback/dummy_lightning_model.py rename to tests/nightly/utils/callbacks/visualizer_callback/dummy_lightning_model.py diff --git a/tests/nightly/core/callbacks/visualizer_callback/test_visualizer.py b/tests/nightly/utils/callbacks/visualizer_callback/test_visualizer.py similarity index 100% rename from tests/nightly/core/callbacks/visualizer_callback/test_visualizer.py rename to tests/nightly/utils/callbacks/visualizer_callback/test_visualizer.py From acd1c7caa82622dadfbf2877511d2281cbaf2805 Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Mon, 7 Feb 2022 13:49:55 +0100 Subject: [PATCH 12/14] =?UTF-8?q?=F0=9F=9A=9A=20Move=20callback=20tests=20?= =?UTF-8?q?to=20pre-merge=20Change=20tox.ini=20to=20reflect=20new=20covera?= =?UTF-8?q?ge=20threshold.=20Ref=20issue=20#94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/helpers/dataset.py | 9 +++++++++ .../utils/callbacks/compress_callback/__init__.py | 0 .../utils/callbacks/compress_callback/dummy_config.yml | 0 .../callbacks/compress_callback/dummy_lightning_model.py | 0 .../utils/callbacks/compress_callback/test_compress.py | 4 ++-- .../utils/callbacks/normalization_callback/__init__.py | 0 .../test_normalization_callback.py | 8 +++++--- .../utils/callbacks/visualizer_callback/__init__.py | 0 .../visualizer_callback/dummy_lightning_model.py | 4 +--- .../callbacks/visualizer_callback/test_visualizer.py | 0 tox.ini | 4 ++-- 11 files changed, 19 insertions(+), 10 deletions(-) rename tests/{nightly => pre_merge}/utils/callbacks/compress_callback/__init__.py (100%) rename tests/{nightly => pre_merge}/utils/callbacks/compress_callback/dummy_config.yml (100%) rename tests/{nightly => pre_merge}/utils/callbacks/compress_callback/dummy_lightning_model.py (100%) rename tests/{nightly => pre_merge}/utils/callbacks/compress_callback/test_compress.py (87%) rename tests/{nightly => pre_merge}/utils/callbacks/normalization_callback/__init__.py (100%) rename tests/{nightly => pre_merge}/utils/callbacks/normalization_callback/test_normalization_callback.py (85%) rename tests/{nightly => pre_merge}/utils/callbacks/visualizer_callback/__init__.py (100%) rename tests/{nightly => pre_merge}/utils/callbacks/visualizer_callback/dummy_lightning_model.py (96%) rename tests/{nightly => pre_merge}/utils/callbacks/visualizer_callback/test_visualizer.py (100%) diff --git a/tests/helpers/dataset.py b/tests/helpers/dataset.py index 97e0a5b117..401704e53d 100644 --- a/tests/helpers/dataset.py +++ b/tests/helpers/dataset.py @@ -82,6 +82,7 @@ def __init__( test_shapes: List[str] = ["hexagon", "star"], path: Union[str, Path] = "./datasets/MVTec", use_mvtec: bool = False, + seed: int = 0, ) -> None: """Creates a context for Generating Dummy Dataset. Useful for wrapping test functions. NOTE: for MVTec dataset it does not return a category. @@ -97,6 +98,7 @@ def __init__( test_shapes (List[str], optional): List of anomalous shapes. Defaults to ["triangle", "ellipse"]. path (Union[str, Path], optional): Path to MVTec dataset. Defaults to "./datasets/MVTec". use_mvtec (bool, optional): Use MVTec dataset or dummy dataset. Defaults to False. + seed (int, optional): Fixes seed if any number greater than 0 is provided. 0 means no seed. Defaults to 0. Example: >>> @TestDataset @@ -112,6 +114,7 @@ def __init__( self.test_shapes = test_shapes self.path = path self.use_mvtec = use_mvtec + self.seed = seed def __call__(self, func): @wraps(func) @@ -129,6 +132,7 @@ def inner(*args, **kwds): train_shapes=self.train_shapes, test_shapes=self.test_shapes, max_size=self.max_size, + seed=self.seed, ) as dataset_path: kwds["category"] = "shapes" return func(*args, path=dataset_path, **kwds) @@ -150,6 +154,7 @@ class GeneratedDummyDataset(ContextDecorator): max_size (Optional[int], optional): Maximum size of the test shapes. Defaults to 10. train_shapes (List[str], optional): List of good shapes. Defaults to ["circle", "rectangle"]. test_shapes (List[str], optional): List of anomalous shapes. Defaults to ["triangle", "ellipse"]. + seed (int, optional): Fixes seed if any number greater than 0 is provided. 0 means no seed. Defaults to 0. """ def __init__( @@ -161,6 +166,7 @@ def __init__( max_size: Optional[int] = 10, train_shapes: List[str] = ["triangle", "rectangle"], test_shapes: List[str] = ["star", "hexagon"], + seed: int = 0, ) -> None: self.root_dir = mkdtemp() self.num_train = num_train @@ -170,6 +176,7 @@ def __init__( self.image_height = img_height self.image_width = img_width self.max_size = max_size + self.seed = seed def _generate_dataset(self): """Generates dummy dataset in a temporary directory using the same @@ -226,6 +233,8 @@ def _generate_dataset(self): def __enter__(self): """Creates the dataset in temp folder.""" + if self.seed > 0: + np.random.seed(self.seed) self._generate_dataset() return self.root_dir diff --git a/tests/nightly/utils/callbacks/compress_callback/__init__.py b/tests/pre_merge/utils/callbacks/compress_callback/__init__.py similarity index 100% rename from tests/nightly/utils/callbacks/compress_callback/__init__.py rename to tests/pre_merge/utils/callbacks/compress_callback/__init__.py diff --git a/tests/nightly/utils/callbacks/compress_callback/dummy_config.yml b/tests/pre_merge/utils/callbacks/compress_callback/dummy_config.yml similarity index 100% rename from tests/nightly/utils/callbacks/compress_callback/dummy_config.yml rename to tests/pre_merge/utils/callbacks/compress_callback/dummy_config.yml diff --git a/tests/nightly/utils/callbacks/compress_callback/dummy_lightning_model.py b/tests/pre_merge/utils/callbacks/compress_callback/dummy_lightning_model.py similarity index 100% rename from tests/nightly/utils/callbacks/compress_callback/dummy_lightning_model.py rename to tests/pre_merge/utils/callbacks/compress_callback/dummy_lightning_model.py diff --git a/tests/nightly/utils/callbacks/compress_callback/test_compress.py b/tests/pre_merge/utils/callbacks/compress_callback/test_compress.py similarity index 87% rename from tests/nightly/utils/callbacks/compress_callback/test_compress.py rename to tests/pre_merge/utils/callbacks/compress_callback/test_compress.py index ffd3a2ee0f..d0cb7c001d 100644 --- a/tests/nightly/utils/callbacks/compress_callback/test_compress.py +++ b/tests/pre_merge/utils/callbacks/compress_callback/test_compress.py @@ -6,7 +6,7 @@ from anomalib.config import get_configurable_parameters from anomalib.core.callbacks.compress import CompressModelCallback -from tests.nightly.utils.callbacks.compress_callback.dummy_lightning_model import ( +from tests.pre_merge.utils.callbacks.compress_callback.dummy_lightning_model import ( DummyLightningModule, FakeDataModule, ) @@ -16,7 +16,7 @@ def test_compress_model_callback(): """Tests if an optimized model is created.""" config = get_configurable_parameters( - model_config_path="tests/nightly/utils/callbacks/compress_callback/dummy_config.yml" + model_config_path="tests/pre_merge/utils/callbacks/compress_callback/dummy_config.yml" ) with tempfile.TemporaryDirectory() as tmp_dir: diff --git a/tests/nightly/utils/callbacks/normalization_callback/__init__.py b/tests/pre_merge/utils/callbacks/normalization_callback/__init__.py similarity index 100% rename from tests/nightly/utils/callbacks/normalization_callback/__init__.py rename to tests/pre_merge/utils/callbacks/normalization_callback/__init__.py diff --git a/tests/nightly/utils/callbacks/normalization_callback/test_normalization_callback.py b/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py similarity index 85% rename from tests/nightly/utils/callbacks/normalization_callback/test_normalization_callback.py rename to tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py index 14237edfda..6874fe4962 100644 --- a/tests/nightly/utils/callbacks/normalization_callback/test_normalization_callback.py +++ b/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py @@ -4,7 +4,7 @@ from anomalib.core.callbacks import get_callbacks from anomalib.data import get_datamodule from anomalib.models import get_model -from tests.helpers.dataset import get_dataset_path +from tests.helpers.dataset import TestDataset, get_dataset_path def run_train_test(config): @@ -17,9 +17,11 @@ def run_train_test(config): return results -def test_normalizer(): +@TestDataset(num_train=100, num_test=20, path=get_dataset_path(), seed=2) +def test_normalizer(path=get_dataset_path(), category="shapes"): config = get_configurable_parameters(model_config_path="anomalib/models/padim/config.yaml") - config.dataset.path = get_dataset_path(config.dataset.path) + config.dataset.path = path + config.dataset.category = category config.model.threshold.adaptive = True # run without normalization diff --git a/tests/nightly/utils/callbacks/visualizer_callback/__init__.py b/tests/pre_merge/utils/callbacks/visualizer_callback/__init__.py similarity index 100% rename from tests/nightly/utils/callbacks/visualizer_callback/__init__.py rename to tests/pre_merge/utils/callbacks/visualizer_callback/__init__.py diff --git a/tests/nightly/utils/callbacks/visualizer_callback/dummy_lightning_model.py b/tests/pre_merge/utils/callbacks/visualizer_callback/dummy_lightning_model.py similarity index 96% rename from tests/nightly/utils/callbacks/visualizer_callback/dummy_lightning_model.py rename to tests/pre_merge/utils/callbacks/visualizer_callback/dummy_lightning_model.py index 04e74cc595..f016dc0825 100644 --- a/tests/nightly/utils/callbacks/visualizer_callback/dummy_lightning_model.py +++ b/tests/pre_merge/utils/callbacks/visualizer_callback/dummy_lightning_model.py @@ -1,8 +1,6 @@ from pathlib import Path -from typing import Tuple, Union -from unittest import mock +from typing import Union -import numpy as np import pytorch_lightning as pl import torch from omegaconf.dictconfig import DictConfig diff --git a/tests/nightly/utils/callbacks/visualizer_callback/test_visualizer.py b/tests/pre_merge/utils/callbacks/visualizer_callback/test_visualizer.py similarity index 100% rename from tests/nightly/utils/callbacks/visualizer_callback/test_visualizer.py rename to tests/pre_merge/utils/callbacks/visualizer_callback/test_visualizer.py diff --git a/tox.ini b/tox.ini index 5389e4b6c5..e38866c345 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,7 @@ deps = commands = coverage erase coverage run --include=anomalib/* -m pytest tests/pre_merge/ -ra - coverage report -m --fail-under=85 + coverage report -m --fail-under=90 coverage xml -o {toxworkdir}/coverage.xml [testenv:nightly] @@ -93,7 +93,7 @@ deps = commands = coverage erase coverage run --include=anomalib/* -m pytest tests/nightly/ -ra - coverage report -m --fail-under=90 + coverage report -m --fail-under=70 coverage xml -o {toxworkdir}/coverage.xml [flake8] From d18c8b688b1d81c20e7bd2a608b64355326c0f13 Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Mon, 7 Feb 2022 15:52:43 +0100 Subject: [PATCH 13/14] Rename setup in test helpers --- tests/helpers/model.py | 18 +++++++++--------- tests/nightly/models/test_model_nightly.py | 4 ++-- tests/pre_merge/models/test_model_premerge.py | 4 ++-- .../test_normalization_callback.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/helpers/model.py b/tests/helpers/model.py index 87a0b2012f..d58aa39456 100644 --- a/tests/helpers/model.py +++ b/tests/helpers/model.py @@ -30,7 +30,7 @@ from anomalib.models import get_model -def setup( +def setup_model_train( model_name: str, dataset_path: str, project_path: str, @@ -88,15 +88,15 @@ def setup( for index, callback in enumerate(callbacks): if isinstance(callback, ModelCheckpoint): callbacks.pop(index) - model_checkpoint = ModelCheckpoint( - dirpath=os.path.join(config.project.path, "weights"), - filename="model", - monitor=None, - mode="max", - auto_insert_metric_name=False, - ) - callbacks.append(model_checkpoint) break + model_checkpoint = ModelCheckpoint( + dirpath=os.path.join(config.project.path, "weights"), + filename="model", + monitor=None, + mode="max", + auto_insert_metric_name=False, + ) + callbacks.append(model_checkpoint) for index, callback in enumerate(callbacks): if isinstance(callback, VisualizerCallback): diff --git a/tests/nightly/models/test_model_nightly.py b/tests/nightly/models/test_model_nightly.py index 91cc040b8a..4151375194 100644 --- a/tests/nightly/models/test_model_nightly.py +++ b/tests/nightly/models/test_model_nightly.py @@ -27,7 +27,7 @@ from anomalib.utils.hpo.config import flatten_sweep_params from tests.helpers.dataset import get_dataset_path -from tests.helpers.model import model_load_test, setup +from tests.helpers.model import model_load_test, setup_model_train def get_model_nncf_cat() -> List: @@ -118,7 +118,7 @@ def test_model(self, model_name, nncf, category, path=get_dataset_path(), score_ seed_everything(42) with tempfile.TemporaryDirectory() as project_path: - config, datamodule, model, trainer = setup( + config, datamodule, model, trainer = setup_model_train( model_name=model_name, dataset_path=path, nncf=nncf, diff --git a/tests/pre_merge/models/test_model_premerge.py b/tests/pre_merge/models/test_model_premerge.py index 5e3c3cae9c..dd55c23f8e 100644 --- a/tests/pre_merge/models/test_model_premerge.py +++ b/tests/pre_merge/models/test_model_premerge.py @@ -20,7 +20,7 @@ import pytest from tests.helpers.dataset import TestDataset -from tests.helpers.model import model_load_test, setup +from tests.helpers.model import model_load_test, setup_model_train class TestModel: @@ -44,7 +44,7 @@ def test_model(self, model_name, nncf, category="shapes", path=""): """Test the models on only 1 epoch as a sanity check before merge.""" with tempfile.TemporaryDirectory() as project_path: # Train test - config, datamodule, model, trainer = setup( + config, datamodule, model, trainer = setup_model_train( model_name, dataset_path=path, project_path=project_path, diff --git a/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py b/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py index fc63b8ac9b..236bdf561a 100644 --- a/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py +++ b/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py @@ -18,7 +18,7 @@ def run_train_test(config): return results -@TestDataset(num_train=100, num_test=20, path=get_dataset_path(), seed=2) +@TestDataset(num_train=100, num_test=20, path=get_dataset_path(), seed=42) def test_normalizer(path=get_dataset_path(), category="shapes"): config = get_configurable_parameters(model_config_path="anomalib/models/padim/config.yaml") config.dataset.path = path From 15cd3ba7ae5a8ca16e28bd317c91bb71ab13c1dd Mon Sep 17 00:00:00 2001 From: Ashwin Vaidya Date: Mon, 7 Feb 2022 17:54:06 +0100 Subject: [PATCH 14/14] =?UTF-8?q?=F0=9F=A9=B9=20Fix=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements/base.txt | 2 +- tests/helpers/model.py | 7 ++++--- .../normalization_callback/test_normalization_callback.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index a800fb288e..13762a497b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -7,7 +7,7 @@ kornia==0.5.6 lxml==4.6.5 matplotlib==3.4.3 networkx~=2.5 -nncf==2.0.0 +nncf==2.1.0 numpy~=1.19.5 omegaconf==2.1.1 pillow==9.0.0 diff --git a/tests/helpers/model.py b/tests/helpers/model.py index d58aa39456..0a9536ac5a 100644 --- a/tests/helpers/model.py +++ b/tests/helpers/model.py @@ -37,7 +37,7 @@ def setup_model_train( nncf: bool, category: str, score_type: str = None, - weight_file: str = "weights/model.ckpt", + weight_file: str = "weights/last.ckpt", fast_run: bool = False, ) -> Tuple[Union[DictConfig, ListConfig], LightningDataModule, AnomalyModule, Trainer]: """Train the model based on the parameters passed. @@ -59,7 +59,7 @@ def setup_model_train( config = get_configurable_parameters(model_name=model_name) if score_type is not None: config.model.score_type = score_type - config.project.seed = 1234 + config.project.seed = 42 config.dataset.category = category config.dataset.path = dataset_path config.project.log_images_to = [] @@ -91,9 +91,10 @@ def setup_model_train( break model_checkpoint = ModelCheckpoint( dirpath=os.path.join(config.project.path, "weights"), - filename="model", + filename="last", monitor=None, mode="max", + save_last=True, auto_insert_metric_name=False, ) callbacks.append(model_checkpoint) diff --git a/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py b/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py index 236bdf561a..399d3d7bd4 100644 --- a/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py +++ b/tests/pre_merge/utils/callbacks/normalization_callback/test_normalization_callback.py @@ -18,7 +18,7 @@ def run_train_test(config): return results -@TestDataset(num_train=100, num_test=20, path=get_dataset_path(), seed=42) +@TestDataset(num_train=200, num_test=30, path=get_dataset_path(), seed=42) def test_normalizer(path=get_dataset_path(), category="shapes"): config = get_configurable_parameters(model_config_path="anomalib/models/padim/config.yaml") config.dataset.path = path