From a344a24899e347e611347b165ab5fcc3a603bbe0 Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Tue, 19 Sep 2023 16:18:08 +0900
Subject: [PATCH 01/17] Fix hook's ordering issue. AdaptiveRepeatHook changes
the runner.max_iters before the ProgressHook
---
.../adapters/mmcls/configurer.py | 7 ----
.../common/adapters/mmcv/clsincr_mixin.py | 8 -----
.../common/adapters/mmcv/configurer.py | 10 +++++-
.../mmcv/hooks/adaptive_repeat_data_hook.py | 34 ++++++++++++++-----
.../adapters/mmcv/hooks/task_adapt_hook.py | 5 ---
.../adapters/mmcv/utils/config_utils.py | 23 +++++++++++++
.../dataloaders/samplers/balanced_sampler.py | 7 ++--
.../dataloaders/samplers/cls_incr_sampler.py | 7 ++--
.../torch/dataloaders/samplers/otx_sampler.py | 32 ++++++++---------
.../multilabel/incremental.yaml | 7 ++--
10 files changed, 85 insertions(+), 55 deletions(-)
diff --git a/src/otx/algorithms/classification/adapters/mmcls/configurer.py b/src/otx/algorithms/classification/adapters/mmcls/configurer.py
index 3199d4cb5d2..4859a38a110 100644
--- a/src/otx/algorithms/classification/adapters/mmcls/configurer.py
+++ b/src/otx/algorithms/classification/adapters/mmcls/configurer.py
@@ -208,13 +208,6 @@ def get_sampler_type(self, cfg):
sampler_type = "cls_incr"
return sampler_type
- def use_adaptive_repeat(self, cfg) -> bool:
- """Return whether using adaptive repeat.
-
- Currently, only multi class classification supports adaptive repeat.
- """
- return self._is_multiclass(cfg)
-
@staticmethod
def _is_multiclass(cfg) -> bool:
return not cfg.model.get("multilabel", False) and not cfg.model.get("hierarchical", False)
diff --git a/src/otx/algorithms/common/adapters/mmcv/clsincr_mixin.py b/src/otx/algorithms/common/adapters/mmcv/clsincr_mixin.py
index bdbdf35e080..02251a8f200 100644
--- a/src/otx/algorithms/common/adapters/mmcv/clsincr_mixin.py
+++ b/src/otx/algorithms/common/adapters/mmcv/clsincr_mixin.py
@@ -36,7 +36,6 @@ def configure_task_adapt_hook(self, cfg):
sampler_flag=sampler_flag,
sampler_type=self.get_sampler_type(cfg),
efficient_mode=cfg["task_adapt"].get("efficient_mode", False),
- use_adaptive_repeat=self.use_adaptive_repeat(cfg),
priority="NORMAL",
),
)
@@ -50,10 +49,3 @@ def is_incremental(self) -> bool:
def get_sampler_type(self, cfg) -> str:
"""Return sampler type."""
return "cls_incr"
-
- def use_adaptive_repeat(self, cfg) -> bool:
- """Return whether using adaptive repeat.
-
- Currently, only multi class classification supports adaptive repeat.
- """
- return False
diff --git a/src/otx/algorithms/common/adapters/mmcv/configurer.py b/src/otx/algorithms/common/adapters/mmcv/configurer.py
index 87ce4b015c9..1c153c1e28e 100644
--- a/src/otx/algorithms/common/adapters/mmcv/configurer.py
+++ b/src/otx/algorithms/common/adapters/mmcv/configurer.py
@@ -16,6 +16,7 @@
patch_adaptive_interval_training,
patch_early_stopping,
patch_persistent_workers,
+ remove_from_configs_by_type,
)
from otx.algorithms.common.adapters.mmcv.utils.config_utils import (
patch_color_conversion,
@@ -195,7 +196,6 @@ def configure_samples_per_gpu(
samples_per_gpu can be changed if it is larger than length of datset
"""
-
for subset in subsets:
if cfg.data.get(subset, None):
dataloader_cfg = cfg.data.get(f"{subset}_dataloader", ConfigDict())
@@ -410,6 +410,14 @@ def configure_hooks(
if hasattr(cfg, "algo_backend"):
self._update_caching_modules(cfg)
+ # Update adaptive repeat
+ if not self.training:
+ remove_from_configs_by_type(cfg.custom_hooks, "AdaptiveRepeatDataHook")
+ for custom_hook in cfg.custom_hooks:
+ if custom_hook["type"] == "AdaptiveRepeatDataHook":
+ custom_hook["train_batch_size"] = cfg.data.train_dataloader.get("samples_per_gpu")
+ custom_hook["n_train_data"] = len(cfg.data.train.get("otx_dataset"))
+
@staticmethod
def _update_caching_modules(cfg: Config) -> None:
def _find_max_num_workers(cfg: dict):
diff --git a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
index 042defcea15..254eabee06d 100644
--- a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
+++ b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
@@ -6,6 +6,7 @@
from mmcv.runner import HOOKS, Hook, get_dist_info
from torch.utils.data import DataLoader
+from otx.algorithms.common.adapters.mmcv.utils.config_utils import get_proper_repeat_times
from otx.algorithms.common.adapters.torch.dataloaders.samplers import OTXSampler
from otx.algorithms.common.utils.logger import get_logger
@@ -17,38 +18,55 @@ class AdaptiveRepeatDataHook(Hook):
"""Hook that adaptively repeats the dataset to control the number of iterations.
Args:
+ train_batch_size (int) : The batch size of the train dataloader
+ n_train_data (int) : The number of the training dataset
coef (float, optional) : coefficient that effects to number of repeats
(coef * math.sqrt(num_iters-1)) +5
min_repeat (float, optional) : minimum repeats
"""
- def __init__(self, coef: float = -0.7, min_repeat: float = 1.0):
+ def __init__(self, train_batch_size: int, n_train_data: int, coef: float = -0.7, min_repeat: float = 1.0):
self.coef = coef
self.min_repeat = min_repeat
+ self.train_batch_size = train_batch_size
+ self.n_train_data = n_train_data
+
+ self.n_repeats = get_proper_repeat_times(self.n_train_data, self.train_batch_size, self.coef, self.min_repeat)
+ self.rank, self.world_size = get_dist_info()
+
+ def before_run(self, runner):
+ """Change the runner's max_iter."""
+ iter_per_epoch = int(self.n_train_data / self.train_batch_size)
+
+ logger.info("Adaptive repeat is enabled")
+ logger.info(f"- Repeat times: {self.n_repeats}")
+ logger.info(f"- Num iters per epoch: {iter_per_epoch} -> {iter_per_epoch * self.n_repeats}")
+ logger.info(f"- Total iters: {runner.max_iters} -> {runner.max_iters * self.n_repeats}")
+
+ runner._max_iters = int(runner.max_iters * self.n_repeats)
+
def before_epoch(self, runner):
"""Convert to OTX Sampler."""
dataset = runner.data_loader.dataset
- batch_size = runner.data_loader.batch_size
num_workers = runner.data_loader.num_workers
collate_fn = runner.data_loader.collate_fn
worker_init_fn = runner.data_loader.worker_init_fn
- rank, world_size = get_dist_info()
sampler = OTXSampler(
dataset=dataset,
- samples_per_gpu=batch_size,
- use_adaptive_repeats=True,
- num_replicas=world_size,
- rank=rank,
+ samples_per_gpu=self.train_batch_size,
+ num_replicas=self.world_size,
+ rank=self.rank,
shuffle=True,
coef=self.coef,
min_repeat=self.min_repeat,
+ n_repeats=self.n_repeats,
)
runner.data_loader = DataLoader(
dataset,
- batch_size=batch_size,
+ batch_size=self.train_batch_size,
sampler=sampler,
num_workers=num_workers,
collate_fn=collate_fn,
diff --git a/src/otx/algorithms/common/adapters/mmcv/hooks/task_adapt_hook.py b/src/otx/algorithms/common/adapters/mmcv/hooks/task_adapt_hook.py
index 468fe0e23a3..29d4f0ad87b 100644
--- a/src/otx/algorithms/common/adapters/mmcv/hooks/task_adapt_hook.py
+++ b/src/otx/algorithms/common/adapters/mmcv/hooks/task_adapt_hook.py
@@ -36,7 +36,6 @@ def __init__(
sampler_flag=False,
sampler_type="cls_incr",
efficient_mode=False,
- use_adaptive_repeat=False,
):
self.src_classes = src_classes
self.dst_classes = dst_classes
@@ -44,13 +43,11 @@ def __init__(
self.sampler_flag = sampler_flag
self.sampler_type = sampler_type
self.efficient_mode = efficient_mode
- self.use_adaptive_repeat = use_adaptive_repeat
logger.info(f"Task Adaptation: {self.src_classes} => {self.dst_classes}")
logger.info(f"- Efficient Mode: {self.efficient_mode}")
logger.info(f"- Sampler type: {self.sampler_type}")
logger.info(f"- Sampler flag: {self.sampler_flag}")
- logger.info(f"- Adaptive repeat: {self.use_adaptive_repeat}")
def before_epoch(self, runner):
"""Produce a proper sampler for task-adaptation."""
@@ -68,7 +65,6 @@ def before_epoch(self, runner):
efficient_mode=self.efficient_mode,
num_replicas=world_size,
rank=rank,
- use_adaptive_repeats=self.use_adaptive_repeat,
)
else:
sampler = ClsIncrSampler(
@@ -77,7 +73,6 @@ def before_epoch(self, runner):
efficient_mode=self.efficient_mode,
num_replicas=world_size,
rank=rank,
- use_adaptive_repeats=self.use_adaptive_repeat,
)
runner.data_loader = DataLoader(
dataset,
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py b/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
index 9a06a5d96f0..2cf044ab6ad 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
@@ -16,6 +16,7 @@
import copy
import glob
+import math
import multiprocessing
import os
import os.path as osp
@@ -887,3 +888,25 @@ def get_configured_input_size(
parsed_tocken = re.match("(\\d+)x(\\d+)", input_size)
return (int(parsed_tocken.group(1)), int(parsed_tocken.group(2)))
+
+
+def get_proper_repeat_times(
+ n_data: int,
+ batch_size: int,
+ coef: float,
+ min_repeat: float,
+) -> float:
+ """Get proper repeat times for adaptive training.
+
+ Args:
+ n_data (int): The total number of the training dataset
+ batch_size (int): The batch size for the training data loader
+ coef (float, optional) : coefficient that effects to number of repeats
+ (coef * math.sqrt(num_iters-1)) +5
+ min_repeat (float, optional) : minimum repeats
+ """
+ if n_data == 0 or batch_size == 0:
+ logger.info("Repeat dataset enabled, but not a train mode. repeat times set to 1.")
+ return 1
+ n_iters_per_epoch = math.ceil(n_data / batch_size)
+ return math.floor(max(coef * math.sqrt(n_iters_per_epoch - 1) + 5, min_repeat))
diff --git a/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/balanced_sampler.py b/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/balanced_sampler.py
index 711f0f4729b..aa12338910c 100644
--- a/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/balanced_sampler.py
+++ b/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/balanced_sampler.py
@@ -4,6 +4,7 @@
#
import math
+from typing import Union
import numpy as np
from torch.utils.data import Dataset
@@ -37,7 +38,7 @@ class BalancedSampler(OTXSampler): # pylint: disable=too-many-instance-attribut
tail of the data to make it evenly divisible across the number of
replicas. If ``False``, the sampler will add extra indices to make
the data evenly divisible across the replicas. Default: ``False``.
- use_adaptive_repeats (bool, optional): Flag about using adaptive repeats
+ n_repeats (Union[float, int, str], optional) : number of iterations for manual setting
"""
def __init__(
@@ -48,14 +49,14 @@ def __init__(
num_replicas: int = 1,
rank: int = 0,
drop_last: bool = False,
- use_adaptive_repeats: bool = False,
+ n_repeats: Union[float, int, str] = "auto",
):
self.samples_per_gpu = samples_per_gpu
self.num_replicas = num_replicas
self.rank = rank
self.drop_last = drop_last
- super().__init__(dataset, samples_per_gpu, use_adaptive_repeats)
+ super().__init__(dataset, samples_per_gpu, n_repeats=n_repeats)
self.img_indices = self.dataset.img_indices # type: ignore[attr-defined]
self.num_cls = len(self.img_indices.keys())
diff --git a/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/cls_incr_sampler.py b/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/cls_incr_sampler.py
index 6b03f8cdf93..ee3fca43699 100644
--- a/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/cls_incr_sampler.py
+++ b/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/cls_incr_sampler.py
@@ -5,6 +5,7 @@
import math
import random
+from typing import Union
import numpy as np
from torch.utils.data import Dataset
@@ -35,7 +36,7 @@ class ClsIncrSampler(OTXSampler): # pylint: disable=too-many-instance-attribute
tail of the data to make it evenly divisible across the number of
replicas. If ``False``, the sampler will add extra indices to make
the data evenly divisible across the replicas. Default: ``False``.
- use_adaptive_repeats (bool, optional): Flag about using adaptive repeats
+ n_repeats (Union[float, int, str], optional) : number of iterations for manual setting
"""
def __init__(
@@ -46,14 +47,14 @@ def __init__(
num_replicas: int = 1,
rank: int = 0,
drop_last: bool = False,
- use_adaptive_repeats: bool = False,
+ n_repeats: Union[float, int, str] = "auto",
):
self.samples_per_gpu = samples_per_gpu
self.num_replicas = num_replicas
self.rank = rank
self.drop_last = drop_last
- super().__init__(dataset, samples_per_gpu, use_adaptive_repeats)
+ super().__init__(dataset, samples_per_gpu, n_repeats=n_repeats)
if hasattr(self.dataset, "img_indices"):
self.new_indices = self.dataset.img_indices["new"]
diff --git a/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/otx_sampler.py b/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/otx_sampler.py
index 9f9a4cac2b6..6af9a269694 100644
--- a/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/otx_sampler.py
+++ b/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/otx_sampler.py
@@ -4,13 +4,14 @@
#
import math
-from typing import Optional
+from typing import Optional, Union
import numpy as np
import torch
from torch.utils.data import Dataset
from torch.utils.data.sampler import Sampler
+from otx.algorithms.common.adapters.mmcv.utils.config_utils import get_proper_repeat_times
from otx.algorithms.common.utils.logger import get_logger
from otx.algorithms.common.utils.task_adapt import unwrap_dataset
@@ -32,7 +33,6 @@ class OTXSampler(Sampler): # pylint: disable=too-many-instance-attributes
Args:
dataset (Dataset): A built-up dataset
samples_per_gpu (int): batch size of Sampling
- use_adaptive_repeats (bool): Flag about using adaptive repeats
num_replicas (int, optional): Number of processes participating in
distributed training. By default, :attr:`world_size` is retrieved from the
current distributed group.
@@ -42,18 +42,22 @@ class OTXSampler(Sampler): # pylint: disable=too-many-instance-attributes
shuffle (bool, optional): Flag about shuffling
coef (int, optional): controls the repeat value
min_repeat (float, optional): minimum value of the repeat dataset
+ n_repeats (Union[float, int str], optional) : number of iterations for manual setting
+ seed (int, optional): Random seed used to shuffle the sampler if
+ :attr:`shuffle=True`. This number should be identical across all
+ processes in the distributed group. Defaults to None.
"""
def __init__(
self,
dataset: Dataset,
samples_per_gpu: int,
- use_adaptive_repeats: bool,
num_replicas: int = 1,
rank: int = 0,
shuffle: bool = True,
coef: float = -0.7,
min_repeat: float = 1.0,
+ n_repeats: Union[float, int, str] = "auto",
seed: Optional[int] = None,
):
@@ -62,8 +66,13 @@ def __init__(
self.num_replicas = num_replicas
self.rank = rank
self.shuffle = shuffle
- self.repeat = self._get_proper_repeats(use_adaptive_repeats, coef, min_repeat)
-
+ if n_repeats == "auto":
+ repeat = get_proper_repeat_times(len(self.dataset), self.samples_per_gpu, coef, min_repeat)
+ elif type(n_repeats).__name__ == "float":
+ repeat = float(n_repeats)
+ else:
+ raise ValueError(f"n_repeats: {n_repeats} should be auto or int value")
+ self.repeat = int(repeat)
self.num_samples = math.ceil(len(self.dataset) * self.repeat / self.num_replicas)
self.total_size = self.num_samples * self.num_replicas
@@ -73,19 +82,6 @@ def __init__(
self.seed = seed
self.epoch = 0
- def _get_proper_repeats(self, use_adaptive_repeats: bool, coef: float, min_repeat: float):
- """Calculate the proper repeats with considering the number of iterations."""
- n_repeats = 1
- if use_adaptive_repeats:
- # NOTE
- # Currently, only support the integer type repeats.
- # Will support the floating point repeats and large dataset cases.
- n_iters_per_epoch = math.ceil(len(self.dataset) / self.samples_per_gpu)
- n_repeats = math.floor(max(coef * math.sqrt(n_iters_per_epoch - 1) + 5, min_repeat))
- logger.info("OTX Sampler: adaptive repeats enabled")
- logger.info(f"OTX will use {n_repeats} times larger dataset made by repeated sampling")
- return n_repeats
-
def __iter__(self):
"""Iter."""
if self.shuffle:
diff --git a/src/otx/recipes/stages/classification/multilabel/incremental.yaml b/src/otx/recipes/stages/classification/multilabel/incremental.yaml
index 85b25d96aa9..cdc23c0eae9 100644
--- a/src/otx/recipes/stages/classification/multilabel/incremental.yaml
+++ b/src/otx/recipes/stages/classification/multilabel/incremental.yaml
@@ -23,5 +23,8 @@ task_adapt:
type: "default_task_adapt"
op: "REPLACE"
-custom_hooks:
- - type: ModelEmaV2Hook
+custom_hooks: [
+ {
+ type: ModelEmaV2Hook
+ }
+]
From 2db44f40119718d18c1a55547822b6c11188bc17 Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Tue, 19 Sep 2023 16:39:48 +0900
Subject: [PATCH 02/17] Change the expression
---
.../adapters/torch/dataloaders/samplers/otx_sampler.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/otx_sampler.py b/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/otx_sampler.py
index 6af9a269694..9abc3d2966a 100644
--- a/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/otx_sampler.py
+++ b/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/otx_sampler.py
@@ -68,10 +68,12 @@ def __init__(
self.shuffle = shuffle
if n_repeats == "auto":
repeat = get_proper_repeat_times(len(self.dataset), self.samples_per_gpu, coef, min_repeat)
- elif type(n_repeats).__name__ == "float":
+ elif isinstance(n_repeats, (int, float)):
repeat = float(n_repeats)
else:
- raise ValueError(f"n_repeats: {n_repeats} should be auto or int value")
+ raise ValueError(f"n_repeats: {n_repeats} should be auto or float orint value")
+ # TODO: Currently, only supporting the int variable.
+ # Will be removed.
self.repeat = int(repeat)
self.num_samples = math.ceil(len(self.dataset) * self.repeat / self.num_replicas)
self.total_size = self.num_samples * self.num_replicas
From 3a0f38f2d12a53dbab0994a02d5420eeeb0a241d Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Tue, 19 Sep 2023 16:45:17 +0900
Subject: [PATCH 03/17] Fix typo
---
.../common/adapters/torch/dataloaders/samplers/otx_sampler.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/otx_sampler.py b/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/otx_sampler.py
index 9abc3d2966a..b01f2aaef66 100644
--- a/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/otx_sampler.py
+++ b/src/otx/algorithms/common/adapters/torch/dataloaders/samplers/otx_sampler.py
@@ -71,7 +71,7 @@ def __init__(
elif isinstance(n_repeats, (int, float)):
repeat = float(n_repeats)
else:
- raise ValueError(f"n_repeats: {n_repeats} should be auto or float orint value")
+ raise ValueError(f"n_repeats: {n_repeats} should be auto or float or int value")
# TODO: Currently, only supporting the int variable.
# Will be removed.
self.repeat = int(repeat)
From d0cf6b2a67295e4ddc13ef9d71cd6492a271d5ea Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Wed, 20 Sep 2023 22:55:12 +0900
Subject: [PATCH 04/17] Fix multi-label, h-label issue
---
.../common/adapters/mmcv/configurer.py | 7 +++--
.../adapters/mmcv/utils/config_utils.py | 14 ++++-----
.../multilabel/incremental.yaml | 29 +++++++++++++++++++
.../classification/multilabel/train.yaml | 2 +-
.../hooks/test_adaptive_repeat_data_hook.py | 4 +--
.../dataloaders/samplers/test_otx_sampler.py | 6 ++--
6 files changed, 44 insertions(+), 18 deletions(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/configurer.py b/src/otx/algorithms/common/adapters/mmcv/configurer.py
index 1c153c1e28e..831f3e7bb84 100644
--- a/src/otx/algorithms/common/adapters/mmcv/configurer.py
+++ b/src/otx/algorithms/common/adapters/mmcv/configurer.py
@@ -415,8 +415,11 @@ def configure_hooks(
remove_from_configs_by_type(cfg.custom_hooks, "AdaptiveRepeatDataHook")
for custom_hook in cfg.custom_hooks:
if custom_hook["type"] == "AdaptiveRepeatDataHook":
- custom_hook["train_batch_size"] = cfg.data.train_dataloader.get("samples_per_gpu")
- custom_hook["n_train_data"] = len(cfg.data.train.get("otx_dataset"))
+ data_cfg = cfg.get("data", {})
+ bs = data_cfg.get("train_dataloader", {}).get("samples_per_gpu", 0)
+ bs = bs if bs is not None else data_cfg.get("samples_per_gpu", 0)
+ custom_hook["train_batch_size"] = bs
+ custom_hook["n_train_data"] = len(data_cfg.get("train",{}).get("otx_dataset",[]))
@staticmethod
def _update_caching_modules(cfg: Config) -> None:
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py b/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
index 2cf044ab6ad..502389a2731 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
@@ -530,15 +530,11 @@ def patch_from_hyperparams(config: Config, hyperparams, **kwargs):
algo_backend = hyperparams.algo_backend
warmup_iters = int(params.learning_rate_warmup_iters)
- model_label_type = config.filename.split("/")[-1]
- if "multilabel" in model_label_type:
- lr_config = ConfigDict(max_lr=params.learning_rate, warmup=None)
- else:
- lr_config = (
- ConfigDict(warmup_iters=warmup_iters)
- if warmup_iters > 0
- else ConfigDict(warmup_iters=warmup_iters, warmup=None)
- )
+ lr_config = (
+ ConfigDict(warmup_iters=warmup_iters)
+ if warmup_iters > 0
+ else ConfigDict(warmup_iters=warmup_iters, warmup=None)
+ )
if params.enable_early_stopping and config.get("evaluation", None):
early_stop = ConfigDict(
diff --git a/src/otx/recipes/stages/classification/multilabel/incremental.yaml b/src/otx/recipes/stages/classification/multilabel/incremental.yaml
index cdc23c0eae9..03a87dc5591 100644
--- a/src/otx/recipes/stages/classification/multilabel/incremental.yaml
+++ b/src/otx/recipes/stages/classification/multilabel/incremental.yaml
@@ -19,12 +19,41 @@ optimizer:
evaluation:
metric: ["accuracy", "class_accuracy"]
+lr_config:
+ _delete_: True
+ policy: ReduceLROnPlateau
+ min_lr: 0.000001
+ interval: 1
+ metric: accuracy
+ factor: 0.5
+ patience: 1
+ iteration_patience: 0
+ warmup: linear
+ warmup_iters: 1
+ warmup_ratio: 0.333
+
task_adapt:
type: "default_task_adapt"
op: "REPLACE"
+ignore: True
+
custom_hooks: [
{
type: ModelEmaV2Hook
+ },
+ {
+ type: LazyEarlyStoppingHook,
+ interval: 1,
+ metric: accuracy,
+ patience: 3,
+ iteration_patience: 0,
+ start: 3,
+ min_delta_ratio: 0.01,
+ priority: 75,
+ },
+ {
+ type: AdaptiveRepeatDataHook,
+ priority: ABOVE_NORMAL
}
]
diff --git a/src/otx/recipes/stages/classification/multilabel/train.yaml b/src/otx/recipes/stages/classification/multilabel/train.yaml
index 5ff73be2aa9..4666a957e36 100644
--- a/src/otx/recipes/stages/classification/multilabel/train.yaml
+++ b/src/otx/recipes/stages/classification/multilabel/train.yaml
@@ -4,7 +4,7 @@ _base_:
"../../_base_/logs/tensorboard_logger.py",
"../../_base_/optimizers/sgd.py",
"../../_base_/runners/epoch_runner_cancel.py",
- "../../_base_/schedules/1cycle.py",
+ "../../_base_/schedules/plateau.py",
]
optimizer:
diff --git a/tests/unit/algorithms/common/adapters/mmcv/hooks/test_adaptive_repeat_data_hook.py b/tests/unit/algorithms/common/adapters/mmcv/hooks/test_adaptive_repeat_data_hook.py
index 17afb05d007..8810e0f1e87 100644
--- a/tests/unit/algorithms/common/adapters/mmcv/hooks/test_adaptive_repeat_data_hook.py
+++ b/tests/unit/algorithms/common/adapters/mmcv/hooks/test_adaptive_repeat_data_hook.py
@@ -21,7 +21,7 @@ def __init__(self):
def __len__(self):
return 10
-
+ self.mock_dataset = MockDataset()
self.mock_data_loader = DataLoader(
dataset=MockDataset(),
batch_size=len(MockDataset()),
@@ -30,7 +30,7 @@ def __len__(self):
@e2e_pytest_unit
def test_before_epoch(self) -> None:
- hook = AdaptiveRepeatDataHook()
+ hook = AdaptiveRepeatDataHook(64, len(self.mock_dataset))
hook.before_epoch(self.mock_runner)
assert self.mock_runner.data_loader.sampler.repeat == 5
diff --git a/tests/unit/algorithms/common/adapters/torch/dataloaders/samplers/test_otx_sampler.py b/tests/unit/algorithms/common/adapters/torch/dataloaders/samplers/test_otx_sampler.py
index 8b72418d72e..6d10da154ab 100644
--- a/tests/unit/algorithms/common/adapters/torch/dataloaders/samplers/test_otx_sampler.py
+++ b/tests/unit/algorithms/common/adapters/torch/dataloaders/samplers/test_otx_sampler.py
@@ -1,5 +1,4 @@
import pytest
-import math
from torch.utils.data import Dataset
from otx.algorithms.common.adapters.torch.dataloaders.samplers import OTXSampler
@@ -20,9 +19,8 @@ def __len__(self):
@e2e_pytest_unit
@pytest.mark.parametrize("batch", [1, 2, 4, 8, 16])
- @pytest.mark.parametrize("use_adaptive_repeat", [True, False])
- def test_sampler_iter(self, batch, use_adaptive_repeat):
- sampler = OTXSampler(self.mock_dataset, batch, use_adaptive_repeats=use_adaptive_repeat)
+ def test_sampler_iter(self, batch):
+ sampler = OTXSampler(self.mock_dataset, batch)
sampler_iter = iter(sampler)
count = 0
From c38dcea34756aad4c2cbb254b18f6baeb4ac36b2 Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Wed, 20 Sep 2023 22:55:45 +0900
Subject: [PATCH 05/17] Make black happy
---
src/otx/algorithms/common/adapters/mmcv/configurer.py | 2 +-
.../adapters/mmcv/hooks/test_adaptive_repeat_data_hook.py | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/configurer.py b/src/otx/algorithms/common/adapters/mmcv/configurer.py
index 831f3e7bb84..56631c1e495 100644
--- a/src/otx/algorithms/common/adapters/mmcv/configurer.py
+++ b/src/otx/algorithms/common/adapters/mmcv/configurer.py
@@ -419,7 +419,7 @@ def configure_hooks(
bs = data_cfg.get("train_dataloader", {}).get("samples_per_gpu", 0)
bs = bs if bs is not None else data_cfg.get("samples_per_gpu", 0)
custom_hook["train_batch_size"] = bs
- custom_hook["n_train_data"] = len(data_cfg.get("train",{}).get("otx_dataset",[]))
+ custom_hook["n_train_data"] = len(data_cfg.get("train", {}).get("otx_dataset", []))
@staticmethod
def _update_caching_modules(cfg: Config) -> None:
diff --git a/tests/unit/algorithms/common/adapters/mmcv/hooks/test_adaptive_repeat_data_hook.py b/tests/unit/algorithms/common/adapters/mmcv/hooks/test_adaptive_repeat_data_hook.py
index 8810e0f1e87..550aea5f611 100644
--- a/tests/unit/algorithms/common/adapters/mmcv/hooks/test_adaptive_repeat_data_hook.py
+++ b/tests/unit/algorithms/common/adapters/mmcv/hooks/test_adaptive_repeat_data_hook.py
@@ -21,6 +21,7 @@ def __init__(self):
def __len__(self):
return 10
+
self.mock_dataset = MockDataset()
self.mock_data_loader = DataLoader(
dataset=MockDataset(),
From 6580630378256d881e5fa7cf3402902fe2302b7a Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Thu, 21 Sep 2023 11:08:03 +0900
Subject: [PATCH 06/17] Fix precommit
---
src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py b/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
index c46ebe85aac..b4ce240214a 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
@@ -974,6 +974,7 @@ def area(x):
logger.info(f"-> Closest preset: {input_size}")
return input_size
+
def get_proper_repeat_times(
n_data: int,
batch_size: int,
From 15cc67d519a71a7a8abef7e6fefdde6e15dc0c3d Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Thu, 21 Sep 2023 14:02:36 +0900
Subject: [PATCH 07/17] Fix auto_bs issue
---
.../algorithms/common/adapters/mmcv/configurer.py | 2 +-
.../mmcv/hooks/adaptive_repeat_data_hook.py | 14 ++++++++------
.../common/adapters/mmcv/utils/automatic_bs.py | 8 ++++++--
3 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/configurer.py b/src/otx/algorithms/common/adapters/mmcv/configurer.py
index 36504e1ef92..c557f09d527 100644
--- a/src/otx/algorithms/common/adapters/mmcv/configurer.py
+++ b/src/otx/algorithms/common/adapters/mmcv/configurer.py
@@ -419,7 +419,7 @@ def configure_hooks(
for custom_hook in cfg.custom_hooks:
if custom_hook["type"] == "AdaptiveRepeatDataHook":
data_cfg = cfg.get("data", {})
- bs = data_cfg.get("train_dataloader", {}).get("samples_per_gpu", 0)
+ bs = data_cfg.get("train_dataloader", {}).get("samples_per_gpu", None)
bs = bs if bs is not None else data_cfg.get("samples_per_gpu", 0)
custom_hook["train_batch_size"] = bs
custom_hook["n_train_data"] = len(data_cfg.get("train", {}).get("otx_dataset", []))
diff --git a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
index 254eabee06d..f3dc49a0f16 100644
--- a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
+++ b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
@@ -37,14 +37,16 @@ def __init__(self, train_batch_size: int, n_train_data: int, coef: float = -0.7,
def before_run(self, runner):
"""Change the runner's max_iter."""
- iter_per_epoch = int(self.n_train_data / self.train_batch_size)
+ if self.n_repeats > 1:
+ iter_per_epoch = int(self.n_train_data / self.train_batch_size)
- logger.info("Adaptive repeat is enabled")
- logger.info(f"- Repeat times: {self.n_repeats}")
- logger.info(f"- Num iters per epoch: {iter_per_epoch} -> {iter_per_epoch * self.n_repeats}")
- logger.info(f"- Total iters: {runner.max_iters} -> {runner.max_iters * self.n_repeats}")
+ logger.info("Adaptive repeat is enabled")
+ logger.info(f"- Repeat times: {self.n_repeats}")
+ logger.info(f"- Batch size: {self.train_batch_size}")
+ logger.info(f"- Num iters per epoch: {iter_per_epoch} -> {iter_per_epoch * self.n_repeats}")
+ logger.info(f"- Total iters: {runner.max_iters} -> {runner.max_iters * self.n_repeats}")
- runner._max_iters = int(runner.max_iters * self.n_repeats)
+ runner._max_iters = int(runner.max_iters * self.n_repeats)
def before_epoch(self, runner):
"""Convert to OTX Sampler."""
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py b/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
index 9b95d58195f..520ff874a8d 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
@@ -68,7 +68,7 @@ def train_func_single_iter(batch_size):
# earlystoppinghook => if eval hook is excluded, this hook makes an error due to absence of score history
# CustomEvalHook => exclude validation in classification task
idx_hooks_to_remove = []
- hooks_to_remove = ["OTXProgressHook", "earlystoppinghook", "CustomEvalHook"]
+ hooks_to_remove = ["OTXProgressHook", "earlystoppinghook", "CustomEvalHook", "AdaptiveRepeatDataHook"]
for i, hook in enumerate(copied_cfg.custom_hooks):
if not validate and hook["type"] == "AdaptiveTrainSchedulingHook":
hook["enable_eval_before_run"] = False
@@ -79,6 +79,7 @@ def train_func_single_iter(batch_size):
if idx_hooks_to_remove:
idx_hooks_to_remove.sort()
for i in reversed(idx_hooks_to_remove):
+ print(f"{copied_cfg.custom_hooks[i]} deleted.")
del copied_cfg.custom_hooks[i]
new_datasets = [SubDataset(datasets[0], batch_size)]
@@ -90,7 +91,6 @@ def train_func_single_iter(batch_size):
)
default_bs = _get_batch_size(cfg)
-
bs_search_algo = BsSearchAlgo(
train_func=train_func_single_iter,
default_bs=default_bs,
@@ -127,6 +127,10 @@ def _set_batch_size(cfg, batch_size: int):
else:
cfg.data.train_dataloader["samples_per_gpu"] = batch_size
+ for custom_hook in cfg.custom_hooks:
+ if custom_hook["type"] == "AdaptiveRepeatDataHook":
+ custom_hook["train_batch_size"] = batch_size
+
def _set_max_epoch(cfg, max_epoch: int):
if cfg.runner.get("type") == "AccuracyAwareRunner": # nncf case
From c0baa7290b841c98899b64fc0a537b312069f3e5 Mon Sep 17 00:00:00 2001
From: Sungman Cho <sungman.cho@intel.com>
Date: Mon, 25 Sep 2023 14:41:34 +0900
Subject: [PATCH 08/17] Apply suggestions from code review
Co-authored-by: Eunwoo Shin <eunwoo.shin@intel.com>
---
src/otx/algorithms/common/adapters/mmcv/configurer.py | 2 ++
src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py | 4 ++--
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/configurer.py b/src/otx/algorithms/common/adapters/mmcv/configurer.py
index c557f09d527..bf648b0379c 100644
--- a/src/otx/algorithms/common/adapters/mmcv/configurer.py
+++ b/src/otx/algorithms/common/adapters/mmcv/configurer.py
@@ -416,6 +416,7 @@ def configure_hooks(
# Update adaptive repeat
if not self.training:
remove_from_configs_by_type(cfg.custom_hooks, "AdaptiveRepeatDataHook")
+ return
for custom_hook in cfg.custom_hooks:
if custom_hook["type"] == "AdaptiveRepeatDataHook":
data_cfg = cfg.get("data", {})
@@ -423,6 +424,7 @@ def configure_hooks(
bs = bs if bs is not None else data_cfg.get("samples_per_gpu", 0)
custom_hook["train_batch_size"] = bs
custom_hook["n_train_data"] = len(data_cfg.get("train", {}).get("otx_dataset", []))
+ break
@staticmethod
def _update_caching_modules(cfg: Config) -> None:
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py b/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
index b4ce240214a..dee79c02242 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
@@ -986,9 +986,9 @@ def get_proper_repeat_times(
Args:
n_data (int): The total number of the training dataset
batch_size (int): The batch size for the training data loader
- coef (float, optional) : coefficient that effects to number of repeats
+ coef (float) : coefficient that effects to number of repeats
(coef * math.sqrt(num_iters-1)) +5
- min_repeat (float, optional) : minimum repeats
+ min_repeat (float) : minimum repeats
"""
if n_data == 0 or batch_size == 0:
logger.info("Repeat dataset enabled, but not a train mode. repeat times set to 1.")
From e1cf24fe04c36af2963f098052c248475a68e2b7 Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Mon, 25 Sep 2023 15:35:15 +0900
Subject: [PATCH 09/17] Reflecting reviews
---
.../common/adapters/mmcv/configurer.py | 2 +-
.../mmcv/hooks/adaptive_repeat_data_hook.py | 60 ++++++++++---------
.../adapters/mmcv/utils/automatic_bs.py | 8 ++-
3 files changed, 39 insertions(+), 31 deletions(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/configurer.py b/src/otx/algorithms/common/adapters/mmcv/configurer.py
index bf648b0379c..96b6e0b4061 100644
--- a/src/otx/algorithms/common/adapters/mmcv/configurer.py
+++ b/src/otx/algorithms/common/adapters/mmcv/configurer.py
@@ -423,7 +423,7 @@ def configure_hooks(
bs = data_cfg.get("train_dataloader", {}).get("samples_per_gpu", None)
bs = bs if bs is not None else data_cfg.get("samples_per_gpu", 0)
custom_hook["train_batch_size"] = bs
- custom_hook["n_train_data"] = len(data_cfg.get("train", {}).get("otx_dataset", []))
+ custom_hook["train_data_size"] = len(self.get_data_cfg(cfg, "train"))
break
@staticmethod
diff --git a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
index f3dc49a0f16..79a7c750b0f 100644
--- a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
+++ b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
@@ -19,26 +19,28 @@ class AdaptiveRepeatDataHook(Hook):
Args:
train_batch_size (int) : The batch size of the train dataloader
- n_train_data (int) : The number of the training dataset
+ train_data_size (int) : The number of the training dataset
coef (float, optional) : coefficient that effects to number of repeats
(coef * math.sqrt(num_iters-1)) +5
min_repeat (float, optional) : minimum repeats
"""
- def __init__(self, train_batch_size: int, n_train_data: int, coef: float = -0.7, min_repeat: float = 1.0):
+ def __init__(self, train_batch_size: int, train_data_size: int, coef: float = -0.7, min_repeat: float = 1.0):
self.coef = coef
self.min_repeat = min_repeat
self.train_batch_size = train_batch_size
- self.n_train_data = n_train_data
+ self.train_data_size =train_data_size
- self.n_repeats = get_proper_repeat_times(self.n_train_data, self.train_batch_size, self.coef, self.min_repeat)
+ self.n_repeats = get_proper_repeat_times(self.train_data_size, self.train_batch_size, self.coef, self.min_repeat)
self.rank, self.world_size = get_dist_info()
+
+ self.is_sampler_changed = False
def before_run(self, runner):
"""Change the runner's max_iter."""
if self.n_repeats > 1:
- iter_per_epoch = int(self.n_train_data / self.train_batch_size)
+ iter_per_epoch = int(self.train_data_size / self.train_batch_size)
logger.info("Adaptive repeat is enabled")
logger.info(f"- Repeat times: {self.n_repeats}")
@@ -50,28 +52,30 @@ def before_run(self, runner):
def before_epoch(self, runner):
"""Convert to OTX Sampler."""
- dataset = runner.data_loader.dataset
- num_workers = runner.data_loader.num_workers
- collate_fn = runner.data_loader.collate_fn
- worker_init_fn = runner.data_loader.worker_init_fn
+ if self.is_sampler_changed == False:
+ dataset = runner.data_loader.dataset
+ num_workers = runner.data_loader.num_workers
+ collate_fn = runner.data_loader.collate_fn
+ worker_init_fn = runner.data_loader.worker_init_fn
- sampler = OTXSampler(
- dataset=dataset,
- samples_per_gpu=self.train_batch_size,
- num_replicas=self.world_size,
- rank=self.rank,
- shuffle=True,
- coef=self.coef,
- min_repeat=self.min_repeat,
- n_repeats=self.n_repeats,
- )
+ sampler = OTXSampler(
+ dataset=dataset,
+ samples_per_gpu=self.train_batch_size,
+ num_replicas=self.world_size,
+ rank=self.rank,
+ shuffle=True,
+ coef=self.coef,
+ min_repeat=self.min_repeat,
+ n_repeats=self.n_repeats,
+ )
- runner.data_loader = DataLoader(
- dataset,
- batch_size=self.train_batch_size,
- sampler=sampler,
- num_workers=num_workers,
- collate_fn=collate_fn,
- pin_memory=False,
- worker_init_fn=worker_init_fn,
- )
+ runner.data_loader = DataLoader(
+ dataset,
+ batch_size=self.train_batch_size,
+ sampler=sampler,
+ num_workers=num_workers,
+ collate_fn=collate_fn,
+ pin_memory=False,
+ worker_init_fn=worker_init_fn,
+ )
+ self.is_sampler_changed = True
\ No newline at end of file
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py b/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
index 520ff874a8d..63392938ed7 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
@@ -11,6 +11,7 @@
from torch.cuda import is_available as cuda_available
from otx.algorithms.common.adapters.torch.utils import BsSearchAlgo
+from otx.algorithms.common.adapters.mmcv.utils.config_utils import update_or_add_custom_hook
from otx.algorithms.common.utils.logger import get_logger
logger = get_logger()
@@ -79,7 +80,7 @@ def train_func_single_iter(batch_size):
if idx_hooks_to_remove:
idx_hooks_to_remove.sort()
for i in reversed(idx_hooks_to_remove):
- print(f"{copied_cfg.custom_hooks[i]} deleted.")
+ print(f"{copied_cfg.custom_hooks[i]} to be deleted.")
del copied_cfg.custom_hooks[i]
new_datasets = [SubDataset(datasets[0], batch_size)]
@@ -126,7 +127,10 @@ def _set_batch_size(cfg, batch_size: int):
cfg.data.videos_per_gpu = batch_size
else:
cfg.data.train_dataloader["samples_per_gpu"] = batch_size
-
+ update_or_add_custom_hook(cfg, {
+ "type": "AdaptiveRepeatDataHook",
+ "train_batch_size": batch_size
+ })
for custom_hook in cfg.custom_hooks:
if custom_hook["type"] == "AdaptiveRepeatDataHook":
custom_hook["train_batch_size"] = batch_size
From 0b3fef8313b5071e434d09130f4aa3585c8a3a96 Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Mon, 25 Sep 2023 16:45:28 +0900
Subject: [PATCH 10/17] Refactor the name of get_data_cfg
---
.../common/adapters/mmcv/configurer.py | 31 ++++++++++---------
.../mmcv/hooks/adaptive_repeat_data_hook.py | 12 ++++---
.../adapters/mmcv/utils/automatic_bs.py | 7 ++---
.../adapters/mmcv/utils/config_utils.py | 8 ++---
.../adapters/mmcv/utils/test_config_utils.py | 16 ++++++++++
5 files changed, 45 insertions(+), 29 deletions(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/configurer.py b/src/otx/algorithms/common/adapters/mmcv/configurer.py
index 96b6e0b4061..3c433ba3232 100644
--- a/src/otx/algorithms/common/adapters/mmcv/configurer.py
+++ b/src/otx/algorithms/common/adapters/mmcv/configurer.py
@@ -112,8 +112,8 @@ def merge_configs(self, cfg, data_cfg, data_pipeline_path, hyperparams_from_otx,
if data_cfg:
for subset in data_cfg.data:
if subset in cfg.data:
- src_data_cfg = self.get_data_cfg(cfg, subset)
- new_data_cfg = self.get_data_cfg(data_cfg, subset)
+ src_data_cfg = self.get_subset_data_cfg(cfg, subset)
+ new_data_cfg = self.get_subset_data_cfg(data_cfg, subset)
for key in new_data_cfg:
src_data_cfg[key] = new_data_cfg[key]
else:
@@ -204,7 +204,7 @@ def configure_samples_per_gpu(
dataloader_cfg = cfg.data.get(f"{subset}_dataloader", ConfigDict())
samples_per_gpu = dataloader_cfg.get("samples_per_gpu", cfg.data.get("samples_per_gpu", 1))
- data_cfg = self.get_data_cfg(cfg, subset)
+ data_cfg = self.get_subset_data_cfg(cfg, subset)
if data_cfg.get("otx_dataset") is not None:
dataset_len = len(data_cfg.otx_dataset)
@@ -269,7 +269,7 @@ def configure_model(self, cfg, data_classes, model_classes, ir_options, **kwargs
self.model_classes = model_classes
self.data_classes = data_classes
if data_classes is not None:
- train_data_cfg = self.get_data_cfg(cfg, "train")
+ train_data_cfg = self.get_subset_data_cfg(cfg, "train")
train_data_cfg["data_classes"] = data_classes
new_classes = np.setdiff1d(data_classes, model_classes).tolist()
train_data_cfg["new_classes"] = new_classes
@@ -417,14 +417,15 @@ def configure_hooks(
if not self.training:
remove_from_configs_by_type(cfg.custom_hooks, "AdaptiveRepeatDataHook")
return
- for custom_hook in cfg.custom_hooks:
- if custom_hook["type"] == "AdaptiveRepeatDataHook":
- data_cfg = cfg.get("data", {})
- bs = data_cfg.get("train_dataloader", {}).get("samples_per_gpu", None)
- bs = bs if bs is not None else data_cfg.get("samples_per_gpu", 0)
- custom_hook["train_batch_size"] = bs
- custom_hook["train_data_size"] = len(self.get_data_cfg(cfg, "train"))
- break
+
+ data_cfg = cfg.get("data", {})
+ bs = data_cfg.get("train_dataloader", {}).get("samples_per_gpu", None)
+ bs = bs if bs is not None else data_cfg.get("samples_per_gpu", 0)
+ train_data_size = len(self.get_subset_data_cfg(cfg, "train").get("otx_dataset", []))
+
+ update_or_add_custom_hook(
+ cfg, {"type": "AdaptiveRepeatDataHook", "train_batch_size": bs, "train_data_size": train_data_size}
+ )
@staticmethod
def _update_caching_modules(cfg: Config) -> None:
@@ -491,7 +492,7 @@ def get_model_meta(cfg):
def get_data_classes(self, cfg):
"""Get data classes from train cfg."""
data_classes = []
- train_cfg = self.get_data_cfg(cfg, "train")
+ train_cfg = self.get_subset_data_cfg(cfg, "train")
if "data_classes" in train_cfg:
data_classes = list(train_cfg.pop("data_classes", []))
elif "classes" in train_cfg:
@@ -499,7 +500,7 @@ def get_data_classes(self, cfg):
return data_classes
@staticmethod
- def get_data_cfg(cfg, subset):
+ def get_subset_data_cfg(cfg, subset):
"""Get subset's data cfg."""
assert subset in ["train", "val", "test", "unlabeled"], f"Unknown subset:{subset}"
if "dataset" in cfg.data[subset]: # Concat|RepeatDataset
@@ -525,7 +526,7 @@ def adapt_input_size_to_dataset(
Tuple[int, int]: (width, height) or None
"""
- data_cfg = BaseConfigurer.get_data_cfg(cfg, "train")
+ data_cfg = BaseConfigurer.get_subset_data_cfg(cfg, "train")
dataset = data_cfg.get("otx_dataset", None)
if dataset is None:
return None
diff --git a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
index 79a7c750b0f..f1633fbbfaf 100644
--- a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
+++ b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
@@ -30,11 +30,13 @@ def __init__(self, train_batch_size: int, train_data_size: int, coef: float = -0
self.min_repeat = min_repeat
self.train_batch_size = train_batch_size
- self.train_data_size =train_data_size
+ self.train_data_size = train_data_size
- self.n_repeats = get_proper_repeat_times(self.train_data_size, self.train_batch_size, self.coef, self.min_repeat)
+ self.n_repeats = get_proper_repeat_times(
+ self.train_data_size, self.train_batch_size, self.coef, self.min_repeat
+ )
self.rank, self.world_size = get_dist_info()
-
+
self.is_sampler_changed = False
def before_run(self, runner):
@@ -52,7 +54,7 @@ def before_run(self, runner):
def before_epoch(self, runner):
"""Convert to OTX Sampler."""
- if self.is_sampler_changed == False:
+ if self.is_sampler_changed is False:
dataset = runner.data_loader.dataset
num_workers = runner.data_loader.num_workers
collate_fn = runner.data_loader.collate_fn
@@ -78,4 +80,4 @@ def before_epoch(self, runner):
pin_memory=False,
worker_init_fn=worker_init_fn,
)
- self.is_sampler_changed = True
\ No newline at end of file
+ self.is_sampler_changed = True
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py b/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
index 63392938ed7..f88d2ef65a6 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
@@ -10,8 +10,8 @@
import numpy as np
from torch.cuda import is_available as cuda_available
-from otx.algorithms.common.adapters.torch.utils import BsSearchAlgo
from otx.algorithms.common.adapters.mmcv.utils.config_utils import update_or_add_custom_hook
+from otx.algorithms.common.adapters.torch.utils import BsSearchAlgo
from otx.algorithms.common.utils.logger import get_logger
logger = get_logger()
@@ -127,10 +127,7 @@ def _set_batch_size(cfg, batch_size: int):
cfg.data.videos_per_gpu = batch_size
else:
cfg.data.train_dataloader["samples_per_gpu"] = batch_size
- update_or_add_custom_hook(cfg, {
- "type": "AdaptiveRepeatDataHook",
- "train_batch_size": batch_size
- })
+ update_or_add_custom_hook(cfg, {"type": "AdaptiveRepeatDataHook", "train_batch_size": batch_size})
for custom_hook in cfg.custom_hooks:
if custom_hook["type"] == "AdaptiveRepeatDataHook":
custom_hook["train_batch_size"] = batch_size
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py b/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
index a310b8441f8..4e99ea291e1 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
@@ -983,7 +983,7 @@ def area(x):
def get_proper_repeat_times(
- n_data: int,
+ data_size: int,
batch_size: int,
coef: float,
min_repeat: float,
@@ -991,14 +991,14 @@ def get_proper_repeat_times(
"""Get proper repeat times for adaptive training.
Args:
- n_data (int): The total number of the training dataset
+ data_size (int): The total number of the training dataset
batch_size (int): The batch size for the training data loader
coef (float) : coefficient that effects to number of repeats
(coef * math.sqrt(num_iters-1)) +5
min_repeat (float) : minimum repeats
"""
- if n_data == 0 or batch_size == 0:
+ if data_size == 0 or batch_size == 0:
logger.info("Repeat dataset enabled, but not a train mode. repeat times set to 1.")
return 1
- n_iters_per_epoch = math.ceil(n_data / batch_size)
+ n_iters_per_epoch = math.ceil(data_size / batch_size)
return math.floor(max(coef * math.sqrt(n_iters_per_epoch - 1) + 5, min_repeat))
diff --git a/tests/unit/algorithms/common/adapters/mmcv/utils/test_config_utils.py b/tests/unit/algorithms/common/adapters/mmcv/utils/test_config_utils.py
index 63d5faf887d..2a7b4014bbe 100644
--- a/tests/unit/algorithms/common/adapters/mmcv/utils/test_config_utils.py
+++ b/tests/unit/algorithms/common/adapters/mmcv/utils/test_config_utils.py
@@ -7,6 +7,7 @@
patch_persistent_workers,
get_adaptive_num_workers,
InputSizeManager,
+ get_proper_repeat_times,
)
from otx.algorithms.common.configs.configuration_enums import InputSizePreset
@@ -491,3 +492,18 @@ def test_adapt_input_size_to_dataset(self):
downscale_only=False,
) # 1024 -> 2048 -> 1024
assert input_size == (1024, 1024)
+
+
+@e2e_pytest_unit
+def test_get_proper_repeat_times():
+ batch_size = 2
+ coef = 1.0
+ min_repeat = 1.0
+
+ data_size = 0
+ repeats = get_proper_repeat_times(data_size=data_size, batch_size=batch_size, coef=coef, min_repeat=min_repeat)
+ assert repeats == 1
+
+ batch_size = 0
+ repeats = get_proper_repeat_times(data_size=data_size, batch_size=batch_size, coef=coef, min_repeat=min_repeat)
+ assert repeats == 1
From af4aeb22d81dbdc437d0799aa0a5bb741383bf1e Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Mon, 25 Sep 2023 16:46:18 +0900
Subject: [PATCH 11/17] Revert adaptive hook sampler init
---
.../mmcv/hooks/adaptive_repeat_data_hook.py | 52 +++++++++----------
1 file changed, 24 insertions(+), 28 deletions(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
index f1633fbbfaf..3ba04893952 100644
--- a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
+++ b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
@@ -37,8 +37,6 @@ def __init__(self, train_batch_size: int, train_data_size: int, coef: float = -0
)
self.rank, self.world_size = get_dist_info()
- self.is_sampler_changed = False
-
def before_run(self, runner):
"""Change the runner's max_iter."""
if self.n_repeats > 1:
@@ -54,30 +52,28 @@ def before_run(self, runner):
def before_epoch(self, runner):
"""Convert to OTX Sampler."""
- if self.is_sampler_changed is False:
- dataset = runner.data_loader.dataset
- num_workers = runner.data_loader.num_workers
- collate_fn = runner.data_loader.collate_fn
- worker_init_fn = runner.data_loader.worker_init_fn
-
- sampler = OTXSampler(
- dataset=dataset,
- samples_per_gpu=self.train_batch_size,
- num_replicas=self.world_size,
- rank=self.rank,
- shuffle=True,
- coef=self.coef,
- min_repeat=self.min_repeat,
- n_repeats=self.n_repeats,
- )
+ dataset = runner.data_loader.dataset
+ num_workers = runner.data_loader.num_workers
+ collate_fn = runner.data_loader.collate_fn
+ worker_init_fn = runner.data_loader.worker_init_fn
+
+ sampler = OTXSampler(
+ dataset=dataset,
+ samples_per_gpu=self.train_batch_size,
+ num_replicas=self.world_size,
+ rank=self.rank,
+ shuffle=True,
+ coef=self.coef,
+ min_repeat=self.min_repeat,
+ n_repeats=self.n_repeats,
+ )
- runner.data_loader = DataLoader(
- dataset,
- batch_size=self.train_batch_size,
- sampler=sampler,
- num_workers=num_workers,
- collate_fn=collate_fn,
- pin_memory=False,
- worker_init_fn=worker_init_fn,
- )
- self.is_sampler_changed = True
+ runner.data_loader = DataLoader(
+ dataset,
+ batch_size=self.train_batch_size,
+ sampler=sampler,
+ num_workers=num_workers,
+ collate_fn=collate_fn,
+ pin_memory=False,
+ worker_init_fn=worker_init_fn,
+ )
From d67b119d43341a5f99378d40d957c0fd14f2d3d9 Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Mon, 25 Sep 2023 16:55:04 +0900
Subject: [PATCH 12/17] Refactor the function name: get_data_cfg ->
get_subset_data_cfg
---
src/otx/algorithms/common/adapters/mmcv/utils/__init__.py | 2 --
.../algorithms/common/adapters/mmcv/utils/config_utils.py | 8 --------
src/otx/algorithms/detection/adapters/mmdet/configurer.py | 2 +-
.../common/adapters/mmcv/utils/test_config_utils.py | 6 +++---
.../detection/adapters/mmdet/test_configurer.py | 4 ++--
5 files changed, 6 insertions(+), 16 deletions(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/__init__.py b/src/otx/algorithms/common/adapters/mmcv/utils/__init__.py
index 4fd056b5725..d6c1ce5a3db 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/__init__.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/__init__.py
@@ -12,7 +12,6 @@
InputSizeManager,
OTXConfig,
config_from_string,
- get_data_cfg,
get_dataset_configs,
is_epoch_based_runner,
patch_adaptive_interval_training,
@@ -45,7 +44,6 @@
"patch_early_stopping",
"patch_persistent_workers",
"prepare_work_dir",
- "get_data_cfg",
"OTXConfig",
"adapt_batch_size",
"InputSizeManager",
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py b/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
index 4e99ea291e1..2b211890232 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py
@@ -596,14 +596,6 @@ def prepare_work_dir(config: Union[Config, ConfigDict]) -> str:
return train_round_checkpoint_dir
-def get_data_cfg(config: Union[Config, ConfigDict], subset: str = "train") -> Config:
- """Return dataset configs."""
- data_cfg = config.data[subset]
- while "dataset" in data_cfg:
- data_cfg = data_cfg.dataset
- return data_cfg
-
-
class InputSizeManager:
"""Class for changing input size and getting input size value by checking data pipeline.
diff --git a/src/otx/algorithms/detection/adapters/mmdet/configurer.py b/src/otx/algorithms/detection/adapters/mmdet/configurer.py
index 8df3ad66d50..876e05ca822 100644
--- a/src/otx/algorithms/detection/adapters/mmdet/configurer.py
+++ b/src/otx/algorithms/detection/adapters/mmdet/configurer.py
@@ -112,7 +112,7 @@ def _configure_eval_dataset(self, cfg):
def configure_task_data_pipeline(self, cfg):
"""Trying to alter class indices of training data according to model class order."""
- tr_data_cfg = self.get_data_cfg(cfg, "train")
+ tr_data_cfg = self.get_subset_data_cfg(cfg, "train")
class_adapt_cfg = dict(type="AdaptClassLabels", src_classes=self.data_classes, dst_classes=self.model_classes)
pipeline_cfg = tr_data_cfg.pipeline
for i, operation in enumerate(pipeline_cfg):
diff --git a/tests/unit/algorithms/common/adapters/mmcv/utils/test_config_utils.py b/tests/unit/algorithms/common/adapters/mmcv/utils/test_config_utils.py
index 2a7b4014bbe..00405854eb9 100644
--- a/tests/unit/algorithms/common/adapters/mmcv/utils/test_config_utils.py
+++ b/tests/unit/algorithms/common/adapters/mmcv/utils/test_config_utils.py
@@ -14,7 +14,7 @@
from tests.test_suite.e2e_test_system import e2e_pytest_unit
-def get_data_cfg(workers_per_gpu: int = 2) -> dict:
+def get_subset_data_cfg(workers_per_gpu: int = 2) -> dict:
data_cfg = {}
for subset in ["train", "val", "test", "unlabeled"]:
data_cfg[subset] = "fake"
@@ -26,7 +26,7 @@ def get_data_cfg(workers_per_gpu: int = 2) -> dict:
@e2e_pytest_unit
@pytest.mark.parametrize("workers_per_gpu", [0, 2])
def test_patch_persistent_workers(mocker, workers_per_gpu):
- data_cfg = get_data_cfg(workers_per_gpu)
+ data_cfg = get_subset_data_cfg(workers_per_gpu)
config = mocker.MagicMock()
config.data = data_cfg
@@ -44,7 +44,7 @@ def test_patch_persistent_workers(mocker, workers_per_gpu):
@e2e_pytest_unit
def test_patch_persistent_workers_dist_semisl(mocker):
- data_cfg = get_data_cfg()
+ data_cfg = get_subset_data_cfg()
config = mocker.MagicMock()
config.data = data_cfg
diff --git a/tests/unit/algorithms/detection/adapters/mmdet/test_configurer.py b/tests/unit/algorithms/detection/adapters/mmdet/test_configurer.py
index 5d5637f81ef..c341d9d1b4e 100644
--- a/tests/unit/algorithms/detection/adapters/mmdet/test_configurer.py
+++ b/tests/unit/algorithms/detection/adapters/mmdet/test_configurer.py
@@ -323,12 +323,12 @@ def test_configure_compat_cfg(self):
self.configurer.configure_compat_cfg(model_cfg)
@e2e_pytest_unit
- def test_get_data_cfg(self):
+ def test_get_subset_data_cfg(self):
config = copy.deepcopy(self.model_cfg)
data_pipeline_cfg = OTXConfig.fromfile(self.data_pipeline_path)
config.merge_from_dict(data_pipeline_cfg)
config.data.train.dataset = ConfigDict({"dataset": [1, 2, 3]})
- assert [1, 2, 3] == self.configurer.get_data_cfg(config, "train")
+ assert [1, 2, 3] == self.configurer.get_subset_data_cfg(config, "train")
class TestIncrDetectionConfigurer:
From 13047348df33c325e73f10e33d69dc47443010c7 Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Tue, 26 Sep 2023 10:19:09 +0900
Subject: [PATCH 13/17] Fix unit test errors
---
.../common/adapters/mmcv/configurer.py | 17 ++++++++---------
.../mmcv/hooks/adaptive_repeat_data_hook.py | 2 ++
.../adapters/mmcls/test_configurer.py | 4 ++--
3 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/configurer.py b/src/otx/algorithms/common/adapters/mmcv/configurer.py
index 3c433ba3232..149fd79f9f5 100644
--- a/src/otx/algorithms/common/adapters/mmcv/configurer.py
+++ b/src/otx/algorithms/common/adapters/mmcv/configurer.py
@@ -417,15 +417,14 @@ def configure_hooks(
if not self.training:
remove_from_configs_by_type(cfg.custom_hooks, "AdaptiveRepeatDataHook")
return
-
- data_cfg = cfg.get("data", {})
- bs = data_cfg.get("train_dataloader", {}).get("samples_per_gpu", None)
- bs = bs if bs is not None else data_cfg.get("samples_per_gpu", 0)
- train_data_size = len(self.get_subset_data_cfg(cfg, "train").get("otx_dataset", []))
-
- update_or_add_custom_hook(
- cfg, {"type": "AdaptiveRepeatDataHook", "train_batch_size": bs, "train_data_size": train_data_size}
- )
+ for custom_hook in cfg.custom_hooks:
+ if custom_hook["type"] == "AdaptiveRepeatDataHook":
+ data_cfg = cfg.get("data", {})
+ bs = data_cfg.get("train_dataloader", {}).get("samples_per_gpu", None)
+ bs = bs if bs is not None else data_cfg.get("samples_per_gpu", 0)
+ custom_hook["train_batch_size"] = bs
+ custom_hook["train_data_size"] = len(data_cfg.get("train", {}).get("otx_dataset", []))
+ break
@staticmethod
def _update_caching_modules(cfg: Config) -> None:
diff --git a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
index 3ba04893952..15526fda9bb 100644
--- a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
+++ b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
@@ -48,6 +48,8 @@ def before_run(self, runner):
logger.info(f"- Num iters per epoch: {iter_per_epoch} -> {iter_per_epoch * self.n_repeats}")
logger.info(f"- Total iters: {runner.max_iters} -> {runner.max_iters * self.n_repeats}")
+ #FIXME, although runner._max_iters is the protected attribute,
+ # There is no way to control the max_iters of runner.
runner._max_iters = int(runner.max_iters * self.n_repeats)
def before_epoch(self, runner):
diff --git a/tests/unit/algorithms/classification/adapters/mmcls/test_configurer.py b/tests/unit/algorithms/classification/adapters/mmcls/test_configurer.py
index 56f1ed27eeb..fd6f26d0805 100644
--- a/tests/unit/algorithms/classification/adapters/mmcls/test_configurer.py
+++ b/tests/unit/algorithms/classification/adapters/mmcls/test_configurer.py
@@ -266,11 +266,11 @@ def test_configure_compat_cfg(self):
self.configurer.configure_compat_cfg(model_cfg)
@e2e_pytest_unit
- def test_get_data_cfg(self):
+ def test_get_subset_data_cfg(self):
config = copy.deepcopy(self.model_cfg)
config.update(self.data_cfg)
config.data.train.dataset = ConfigDict({"dataset": [1, 2, 3]})
- assert [1, 2, 3] == self.configurer.get_data_cfg(config, "train")
+ assert [1, 2, 3] == self.configurer.get_subset_data_cfg(config, "train")
class TestIncrClassificationConfigurer:
From 334609568695a8f5ed3ac63d17964113c552d0a5 Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Tue, 26 Sep 2023 10:28:39 +0900
Subject: [PATCH 14/17] Fix black
---
.../common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
index 15526fda9bb..310e7316ba6 100644
--- a/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
+++ b/src/otx/algorithms/common/adapters/mmcv/hooks/adaptive_repeat_data_hook.py
@@ -48,8 +48,8 @@ def before_run(self, runner):
logger.info(f"- Num iters per epoch: {iter_per_epoch} -> {iter_per_epoch * self.n_repeats}")
logger.info(f"- Total iters: {runner.max_iters} -> {runner.max_iters * self.n_repeats}")
- #FIXME, although runner._max_iters is the protected attribute,
- # There is no way to control the max_iters of runner.
+ # FIXME, although runner._max_iters is the protected attribute,
+ # There is no way to control the max_iters of runner.
runner._max_iters = int(runner.max_iters * self.n_repeats)
def before_epoch(self, runner):
From 34b8f7157f6a79d8ec740b6cf22ebd6151681d2b Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Wed, 4 Oct 2023 13:13:59 +0900
Subject: [PATCH 15/17] Remove useless line
---
src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py b/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
index f88d2ef65a6..400558ceee4 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
@@ -80,7 +80,6 @@ def train_func_single_iter(batch_size):
if idx_hooks_to_remove:
idx_hooks_to_remove.sort()
for i in reversed(idx_hooks_to_remove):
- print(f"{copied_cfg.custom_hooks[i]} to be deleted.")
del copied_cfg.custom_hooks[i]
new_datasets = [SubDataset(datasets[0], batch_size)]
From 040bec85652892574d8d4a9818db48cc0da3f306 Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Wed, 4 Oct 2023 13:36:03 +0900
Subject: [PATCH 16/17] Remove adding AdaptiveRepeatDataHook for autobs
---
src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py b/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
index 400558ceee4..637368bd557 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
@@ -69,7 +69,7 @@ def train_func_single_iter(batch_size):
# earlystoppinghook => if eval hook is excluded, this hook makes an error due to absence of score history
# CustomEvalHook => exclude validation in classification task
idx_hooks_to_remove = []
- hooks_to_remove = ["OTXProgressHook", "earlystoppinghook", "CustomEvalHook", "AdaptiveRepeatDataHook"]
+ hooks_to_remove = ["OTXProgressHook", "earlystoppinghook", "CustomEvalHook"]
for i, hook in enumerate(copied_cfg.custom_hooks):
if not validate and hook["type"] == "AdaptiveTrainSchedulingHook":
hook["enable_eval_before_run"] = False
@@ -126,7 +126,6 @@ def _set_batch_size(cfg, batch_size: int):
cfg.data.videos_per_gpu = batch_size
else:
cfg.data.train_dataloader["samples_per_gpu"] = batch_size
- update_or_add_custom_hook(cfg, {"type": "AdaptiveRepeatDataHook", "train_batch_size": batch_size})
for custom_hook in cfg.custom_hooks:
if custom_hook["type"] == "AdaptiveRepeatDataHook":
custom_hook["train_batch_size"] = batch_size
From 8518fbd6d1c8a37b98b5f0d61cee4148e3d274a9 Mon Sep 17 00:00:00 2001
From: sungmanc <sungman.cho@intel.com>
Date: Wed, 4 Oct 2023 14:53:08 +0900
Subject: [PATCH 17/17] Remove unused import
---
src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py b/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
index 637368bd557..3ccd2c81f3f 100644
--- a/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
+++ b/src/otx/algorithms/common/adapters/mmcv/utils/automatic_bs.py
@@ -10,7 +10,6 @@
import numpy as np
from torch.cuda import is_available as cuda_available
-from otx.algorithms.common.adapters.mmcv.utils.config_utils import update_or_add_custom_hook
from otx.algorithms.common.adapters.torch.utils import BsSearchAlgo
from otx.algorithms.common.utils.logger import get_logger