diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c103706da..61842696bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,27 +8,79 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added -- πŸš€ Update OpenVINO and ONNX export to support fixed input shape by @adrianboguszewski in https://github.com/openvinotoolkit/anomalib/pull/2006 -- Add data_path argument to predict entrypoint and add properties for retrieving model path by @djdameln in https://github.com/openvinotoolkit/anomalib/pull/2018 +### Changed + +### Deprecated + +### Fixed + +### New Contributors + +**Full Changelog**: + +## [v1.1.0] + +### Added + +- πŸš€ Add support for MLFlow logger by @DoMaLi94 in https://github.com/openvinotoolkit/anomalib/pull/1847 +- πŸ“š Add Transform behaviour+documentation by @djdameln in https://github.com/openvinotoolkit/anomalib/pull/1953 +- πŸ“š Add documentation on how to use the tiler by @blaz-r in https://github.com/openvinotoolkit/anomalib/pull/1960 +- πŸ’¬ Add Discord badge to `README.md` by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/2012 +- πŸš€ Add Auto-Encoder based FRE by @nahuja-intel in https://github.com/openvinotoolkit/anomalib/pull/2025 - πŸš€ Add compression and quantization for OpenVINO export by @adrianboguszewski in https://github.com/openvinotoolkit/anomalib/pull/2052 -- πŸš€from_config API: Create a path between API & configuration file (CLI) by @harimkang in https://github.com/openvinotoolkit/anomalib/pull/2065 +- πŸš€ Add Anomalib Pipelines by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/2060 +- πŸš€ Add `from_config` API: Create a path between API & configuration file (CLI) by @harimkang in https://github.com/openvinotoolkit/anomalib/pull/2065 +- πŸš€ Add data filter in tar extract by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/2084 ### Changed -- WinCLIP: set device in text embedding collection and apply forward pass with no grad, by @djdameln in https://github.com/openvinotoolkit/anomalib/pull/1984 - πŸ”¨ Move all export functionalities to AnomalyModule as base methods by @thinhngo-x in () -- Remove unnecessary jsonargparse dependencies by @davnn in () -- Use default model-specific eval transform when only train_transform specified by @djdameln(https://github.com/djdameln) in () -- πŸ”¨Rename OptimalF1 to F1Max for consistency with the literature, by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/1980 -- 🐞Update OptimalF1 score to use BinaryPrecisionRecallCurve and remove num_classes by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/1972 +- ⬆️ Update torch and lightning package versions by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/1949 +- πŸ”¨ Use default model-specific eval transform when only train_transform specified by @djdameln(https://github.com/djdameln) in () +- πŸ”¨ Replace `@abstractproperty` since it is deprecated by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/1964 +- πŸ› οΈ Update OptimalF1 Score by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/1972 +- πŸ”¨ Rename OptimalF1 to F1Max for consistency with the literature, by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/1980 +- πŸ”¨ WinCLIP: set device in text embedding collection and apply forward pass with no grad, by @djdameln in https://github.com/openvinotoolkit/anomalib/pull/1984 +- πŸ”¨ WinCLIP improvements by @djdameln in https://github.com/openvinotoolkit/anomalib/pull/1985 +- πŸš€ Update OpenVINO and ONNX export to support fixed input shape by @adrianboguszewski in https://github.com/openvinotoolkit/anomalib/pull/2006 +- πŸ”¨ Update lightning inference by @djdameln in https://github.com/openvinotoolkit/anomalib/pull/2018 +- ⬆️ Upgrade wandb by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/2040 +- πŸ”¨ Refactor Export by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/2057 +- ⬆️ Update `pyproject.toml` so `liccheck` can pick the license by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/2074 +- ⬆️ Update timm requirement from <=0.9.16,>=0.5.4 to >=0.5.4,<=1.0.3 by @dependabot in https://github.com/openvinotoolkit/anomalib/pull/2075 +- πŸ”¨ Update model `README.md` files by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/2076 ### Deprecated +- πŸ—‘οΈ Remove labeler and update codeowners by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/1946 +- πŸ—‘οΈ Remove requirements directory by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/1945 +- πŸ—‘οΈ Remove Docker related files by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/2039 +- πŸ—‘οΈ Remove references to nightly tests by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/2048 +- πŸ—‘οΈ Remove unnecessary jsonargparse dependencies by @davnn in https://github.com/openvinotoolkit/anomalib/pull/2046 + ### Fixed +- 🐞 Fix dimensionality mismatch issue caused by the new kornia version by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/1944 +- 🐞 Fix DFM PyTorch inference by @adrianboguszewski in https://github.com/openvinotoolkit/anomalib/pull/1952 +- 🐞 Fix anomaly map shape to also work with tiling by @blaz-r in https://github.com/openvinotoolkit/anomalib/pull/1959 +- 🐞 Fix EfficientAD's pretrained weigths load path by @seyeon923 in https://github.com/openvinotoolkit/anomalib/pull/1966 +- 🐞 fixbug: use BinaryPrecisionRecallCurve instead of PrecisionRecallCurve by @rglkt in https://github.com/openvinotoolkit/anomalib/pull/1956 +- 🚨 Hotfix: compute precision recall on raw scores by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/1973 +- 🐞 Minor fix to remove input_size from Padim config by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/1988 +- 🐞 Fix Reverse Distillation export to ONNX by @adrianboguszewski in https://github.com/openvinotoolkit/anomalib/pull/1990 +- 🐞 Fix DSR training when no GPU by @adrianboguszewski in https://github.com/openvinotoolkit/anomalib/pull/2004 +- 🐞 Fix efficient ad by @abc-125 in https://github.com/openvinotoolkit/anomalib/pull/2015 +- 🐞 Fix keys in data configs to fit AnomalibDataModule parameters by @abc-125 in https://github.com/openvinotoolkit/anomalib/pull/2032 +- 🐞 Fix Export docstring in CLI by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/2058 +- 🐞 Fix UFlow links by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/2059 + ### New Contributors -**Full Changelog**: +- @seyeon923 made their first contribution in https://github.com/openvinotoolkit/anomalib/pull/1966 +- @rglkt made their first contribution in https://github.com/openvinotoolkit/anomalib/pull/1956 +- @DoMaLi94 made their first contribution in https://github.com/openvinotoolkit/anomalib/pull/1847 + +**Full Changelog**: https://github.com/openvinotoolkit/anomalib/compare/v1.0.1...v1.1.0 ## [v1.0.1] - 2024-03-27 @@ -124,7 +176,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - πŸ”’ Replace `md5` with `sha-256` by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/1680 - πŸ”¨ Refactor Visualisation by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/1693 - πŸš€ Replace `albumentations` with `torchvision` transforms by @djdameln in https://github.com/openvinotoolkit/anomalib/pull/1706 -- πŸ’₯ Create a script to upgrade v0.\* configuration format to v1 by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/1738 +- πŸ’₯ Create a script to upgrade v0.\- configuration format to v1 by @samet-akcay in https://github.com/openvinotoolkit/anomalib/pull/1738 - πŸ”¨ Refactor type alias by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/1742 - πŸ”¨ Remove Lightning dependencies from the CLI and Add `anomalib install` subcommand by @harimkang in https://github.com/openvinotoolkit/anomalib/pull/1748 - πŸ”¨ Refactor `Engine.predict` method by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/1772 @@ -376,7 +428,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configure reference frame for multi-frame video clips () - Bump OpenVINO version to `2022.3.0` () - Remove the dependecy on a specific `torchvision` and `torchmetrics` packages. -- Bump PyTorch Lightning version to v.1.9.\* () +- Bump PyTorch Lightning version to v.1.9.\- () - Make input image normalization and center cropping configurable from config (https://github.com/openvinotoolkit/anomalib/pull/822) - Improve flexibility and configurability of subset splitting (https://github.com/openvinotoolkit/anomalib/pull/822) - Switch to new datamodules design (https://github.com/openvinotoolkit/anomalib/pull/822) diff --git a/docs/source/markdown/guides/reference/models/image/index.md b/docs/source/markdown/guides/reference/models/image/index.md index e52dd24ed9..2229a3b15d 100644 --- a/docs/source/markdown/guides/reference/models/image/index.md +++ b/docs/source/markdown/guides/reference/models/image/index.md @@ -114,6 +114,8 @@ Student-Teacher Feature Pyramid Matching for Unsupervised Anomaly Detection :link-type: doc U-Flow: A U-shaped Normalizing Flow for Anomaly Detection with Unsupervised Threshold +::: + :::{grid-item-card} {material-regular}`model_training;1.5em` WinCLIP :link: ./winclip :link-type: doc diff --git a/docs/source/markdown/guides/reference/pipelines/base/generator.md b/docs/source/markdown/guides/reference/pipelines/base/generator.md new file mode 100644 index 0000000000..0249b06600 --- /dev/null +++ b/docs/source/markdown/guides/reference/pipelines/base/generator.md @@ -0,0 +1,9 @@ +# Generator + +```{eval-rst} +.. autoclass:: anomalib.pipelines.components.base.job.JobGenerator + :members: + :inherited-members: + :show-inheritance: + +``` diff --git a/docs/source/markdown/guides/reference/pipelines/base/job.md b/docs/source/markdown/guides/reference/pipelines/base/job.md new file mode 100644 index 0000000000..988516b5a7 --- /dev/null +++ b/docs/source/markdown/guides/reference/pipelines/base/job.md @@ -0,0 +1,9 @@ +# Job + +```{eval-rst} +.. autoclass:: anomalib.pipelines.components.base.job.Job + :members: + :inherited-members: + :show-inheritance: + +``` diff --git a/docs/source/markdown/guides/reference/pipelines/base/pipeline.md b/docs/source/markdown/guides/reference/pipelines/base/pipeline.md new file mode 100644 index 0000000000..dc9f11c743 --- /dev/null +++ b/docs/source/markdown/guides/reference/pipelines/base/pipeline.md @@ -0,0 +1,9 @@ +# Pipeline Base Class + +The `Pipeline` class is the base class for all pipelines. It provides the following methods: + +```{eval-rst} +.. automodule:: anomalib.pipelines.components.base.pipeline + :members: + :show-inheritance: +``` diff --git a/docs/source/markdown/guides/reference/pipelines/benchmark/generator.md b/docs/source/markdown/guides/reference/pipelines/benchmark/generator.md new file mode 100644 index 0000000000..e8e9a0f4ac --- /dev/null +++ b/docs/source/markdown/guides/reference/pipelines/benchmark/generator.md @@ -0,0 +1,10 @@ +# Benchmark Job Generator + +```{eval-rst} + +.. autoclass:: anomalib.pipelines.benchmark.generator.BenchmarkJobGenerator + :members: + :inherited-members: + :show-inheritance: + +``` diff --git a/docs/source/markdown/guides/reference/pipelines/benchmark/index.md b/docs/source/markdown/guides/reference/pipelines/benchmark/index.md new file mode 100644 index 0000000000..93bae8627e --- /dev/null +++ b/docs/source/markdown/guides/reference/pipelines/benchmark/index.md @@ -0,0 +1,91 @@ +# Benchmarking Pipeline + +The benchmarking pipeline allows you to run multiple models across combination of parameters and dataset categories to collect metrics. The benchmarking run is configured using a config file that specifies the grid-search parameters. A sample config file is shown below: + +```yaml +accelerator: + - cuda + - cpu +benchmark: + seed: 42 + model: + class_path: + grid_search: [Padim, Patchcore] + data: + class_path: MVTec + init_args: + category: + grid: + - bottle + - cable + - capsule +``` + +The `accelerator` parameter is specific to the pipeline and is used to configure the runners. When `cuda` is passed it adds a [parallel](../runners/parallel.md) runner with number of jobs equal to the number of cuda devices. The idea is that since job is independent, we can increase the throughput by distributing each on an individual accelerator. The `cpu` jobs are run [serially](../runners/serial.md). + +## Running the Benchmark Pipeline + +There are two ways to run the benchmark pipeline; as a subcommand, or as a standalone entrypoint. + +:::::{dropdown} CLI +:icon: code + +::::{tab-set} +:::{tab-item} Anomalib subcommand +:sync: label-1 + +```{literalinclude} ../../../../../snippets/pipelines/benchmark/cli_anomalib.txt +:language: bash +``` + +::: + +:::{tab-item} Standalone entrypoint +:sync: label-2 + +```{literalinclude} ../../../../../snippets/pipelines/benchmark/cli_tools.txt +:language: bash +``` + +::: + +::::: + +## Benchmark Pipeline Class + +```{eval-rst} + +.. autoclass:: anomalib.pipelines.benchmark.pipeline.Benchmark + :members: + :inherited-members: + :show-inheritance: + +``` + +::::{grid} +:margin: 1 1 0 0 +:gutter: 1 + +:::{grid-item-card} Job +:link: ./job +:link-type: doc + +Benchmark Job +::: + +:::{grid-item-card} Generator +:link: ./generator +:link-type: doc + +Benchmark Job Generator +::: + +:::: + +```{toctree} +:caption: Benchmark +:hidden: + +./job +./generator +``` diff --git a/docs/source/markdown/guides/reference/pipelines/benchmark/job.md b/docs/source/markdown/guides/reference/pipelines/benchmark/job.md new file mode 100644 index 0000000000..d445c6c931 --- /dev/null +++ b/docs/source/markdown/guides/reference/pipelines/benchmark/job.md @@ -0,0 +1,10 @@ +# Benchmark Job + +```{eval-rst} + +.. autoclass:: anomalib.pipelines.benchmark.job.BenchmarkJob + :members: + :inherited-members: + :show-inheritance: + +``` diff --git a/docs/source/markdown/guides/reference/pipelines/index.md b/docs/source/markdown/guides/reference/pipelines/index.md new file mode 100644 index 0000000000..09537aa39c --- /dev/null +++ b/docs/source/markdown/guides/reference/pipelines/index.md @@ -0,0 +1,83 @@ +# Pipelines + +```{danger} +The pipelines feature is experimental and might be changed without backward compatibility. +``` + +## Introduction + +Tasks such as Benchmarking, Ensemble Tiling, and Hyper-parameter optimization requires running multiple models and chaining multiple stages together. The pipelines feature provides a way to define and run such tasks. Each part of the pipeline is designed to be independent and composable so that they can be reused across different pipelines. + +## Terminology + +- **Pipeline**: Pipeline is the main entity that defines the sequence of [jobs](./base/job.md) to be executed. It is responsible for creating and running the jobs. The job itself is generated using a [job generator](./base/generator.md). And, these are chained using a [runner](./runners/index.md). + +- **Runner**: A runner is responsible for scheduling and running the jobs. It also passes the output of the previous job, if available. It also calls the right hooks to gather and save the results from the jobs and passes the gathered results to the next runner. + +- **Job Generator**: The job generator is responsible for generating jobs based on the configuration. It is used by the runner to create jobs. + +- **Job**: A job is an atomic unit of work that can be run independently. It is responsible for running a single task. For example, training a model or computing metrics. The idea behind this is to ensure that it can be attached to any runner without making changes to the job itself. This is useful when you want to distribute the jobs to increase the throughput of your pipeline. + +```{admonition} Detailed Walkthrough +:class: tip +For more clarity on creating a custom pipeline, refer to the [How-To Guide](../../how_to/pipelines/index.md). +``` + +## Base classes + +::::{grid} +:margin: 1 1 0 0 +:gutter: 1 + +:::{grid-item-card} {octicon}`workflow` Pipeline +:link: ./base/pipeline +:link-type: doc + +Base class for pipeline. +::: + +:::{grid-item-card} {octicon}`file` Job +:link: ./base/job +:link-type: doc + +Base class for job. +::: + +:::{grid-item-card} {octicon}`iterations` Job Generator +:link: ./base/generator +:link-type: doc + +Base class for job generator. +::: + +:::{grid-item-card} {octicon}`play` Runner +:link: ./runners/index +:link-type: doc + +Base class for runner. +::: + +:::: + +## Available Pipelines + +::::{grid} +:margin: 1 1 0 0 +:gutter: 1 + +:::{grid-item-card} {octicon}`number` Benchmarking +:link: ./benchmark/index +:link-type: doc + +Compute metrics across models using a grid-search. +::: + +:::: + +```{toctree} +:caption: Pipelines +:hidden: + +./benchmark/index +./runners/index +``` diff --git a/docs/source/markdown/guides/reference/pipelines/runners/index.md b/docs/source/markdown/guides/reference/pipelines/runners/index.md new file mode 100644 index 0000000000..374eec4852 --- /dev/null +++ b/docs/source/markdown/guides/reference/pipelines/runners/index.md @@ -0,0 +1,39 @@ +# Runner + +```{eval-rst} +.. autoclass:: anomalib.pipelines.components.base.runner.Runner + :members: + +``` + +## Available Runners + +Anomalib provides a few runners that can be used in your pipelines. + +::::{grid} +:margin: 1 1 0 0 +:gutter: 1 + +:::{grid-item-card} {octicon}`list-ordered` Serial Runner +:link: ./serial +:link-type: doc + +Runner for serial jobs. +::: + +:::{grid-item-card} {octicon}`git-branch` Parallel Runner +:link: ./parallel +:link-type: doc + +Runner for parallel jobs. +::: + +:::: + +```{toctree} +:caption: Runners +:hidden: + +./serial +./parallel +``` diff --git a/docs/source/markdown/guides/reference/pipelines/runners/parallel.md b/docs/source/markdown/guides/reference/pipelines/runners/parallel.md new file mode 100644 index 0000000000..9e1c9e83b2 --- /dev/null +++ b/docs/source/markdown/guides/reference/pipelines/runners/parallel.md @@ -0,0 +1,7 @@ +# Parallel Runner + +```{eval-rst} +.. automodule:: anomalib.pipelines.components.runners.parallel + :members: + :show-inheritance: +``` diff --git a/docs/source/markdown/guides/reference/pipelines/runners/serial.md b/docs/source/markdown/guides/reference/pipelines/runners/serial.md new file mode 100644 index 0000000000..d9fdb89e32 --- /dev/null +++ b/docs/source/markdown/guides/reference/pipelines/runners/serial.md @@ -0,0 +1,7 @@ +# Serial Runner + +```{eval-rst} +.. automodule:: anomalib.pipelines.components.runners.serial + :members: + :show-inheritance: +``` diff --git a/docs/source/snippets/pipelines/benchmark/cli_anomalib.txt b/docs/source/snippets/pipelines/benchmark/cli_anomalib.txt new file mode 100644 index 0000000000..e28dc5cb22 --- /dev/null +++ b/docs/source/snippets/pipelines/benchmark/cli_anomalib.txt @@ -0,0 +1,2 @@ +# Using Anomalib entrypoint +anomalib benchmark --config tools/experimental/benchmarking/sample.yaml diff --git a/docs/source/snippets/pipelines/benchmark/cli_tools.txt b/docs/source/snippets/pipelines/benchmark/cli_tools.txt new file mode 100644 index 0000000000..df86e64ac3 --- /dev/null +++ b/docs/source/snippets/pipelines/benchmark/cli_tools.txt @@ -0,0 +1,2 @@ +# Using Entrypoint in tools +python tools/experimental/benchmarking/benchmark.py --config tools/experimental/benchmarking/sample.yaml diff --git a/src/anomalib/__init__.py b/src/anomalib/__init__.py index 711eb023e9..0ca656804e 100644 --- a/src/anomalib/__init__.py +++ b/src/anomalib/__init__.py @@ -5,7 +5,7 @@ from enum import Enum -__version__ = "1.1.0dev" +__version__ = "1.1.0" class LearningType(str, Enum): diff --git a/src/anomalib/pipelines/components/base/runner.py b/src/anomalib/pipelines/components/base/runner.py index 54fa0c5f31..cee46dfacb 100644 --- a/src/anomalib/pipelines/components/base/runner.py +++ b/src/anomalib/pipelines/components/base/runner.py @@ -11,11 +11,35 @@ class Runner(ABC): - """Base runner.""" + """Base runner. + + Args: + generator (JobGenerator): Job generator. + """ def __init__(self, generator: JobGenerator) -> None: self.generator = generator @abstractmethod def run(self, args: dict, prev_stage_results: PREV_STAGE_RESULT = None) -> GATHERED_RESULTS: - """Run the pipeline.""" + """Run the pipeline. + + Args: + args (dict): Arguments specific to the job. For example, if there is a pipeline defined where one of the job + generators is hyperparameter optimization, then the pipeline configuration file will look something like + ```yaml + arg1: + arg2: + hpo: + param1: + param2: + ... + ``` + In this case, the `args` will receive a dictionary with all keys under `hpo`. + + prev_stage_results (PREV_STAGE_RESULT, optional): Previous stage results. This is useful when the current + stage depends on the results of the previous stage. Defaults to None. + + Returns: + GATHERED_RESULTS: Gathered results from all the jobs. + """