From 70a8f3a96a0d3bd25fdd70e19c190ded30a953b5 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Tue, 14 Jul 2020 16:49:04 +0800 Subject: [PATCH 1/3] update import --- .../gradient_selector/fginitialize.py | 40 +++++++------------ .../gradient_selector/learnability.py | 5 +-- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/fginitialize.py b/src/sdk/pynni/nni/feature_engineering/gradient_selector/fginitialize.py index 3a5bf9ee5a..6fe28ea5ee 100644 --- a/src/sdk/pynni/nni/feature_engineering/gradient_selector/fginitialize.py +++ b/src/sdk/pynni/nni/feature_engineering/gradient_selector/fginitialize.py @@ -31,7 +31,7 @@ import torch from torch.utils.data import DataLoader, Dataset # pylint: disable=E0611 -from torch.utils.data.dataloader import _DataLoaderIter, _utils +from torch.utils.data.dataloader import _SingleProcessDataLoaderIter, _MultiProcessingDataLoaderIter, _utils from . import constants from . import syssettings @@ -585,39 +585,27 @@ def __iter__(self): return _ChunkDataLoaderIter(self) -class _ChunkDataLoaderIter(_DataLoaderIter): +class _ChunkDataLoaderIter: """ DataLoaderIter class used to more quickly load a batch of indices at once. """ + def __init__(self, dataloader): + if dataloader.num_workers == 0: + self.iter = _SingleProcessDataLoaderIter(dataloader) + else: + self.iter = _MultiProcessingDataLoaderIter(dataloader) def __next__(self): # only chunk that is edited from base - if self.num_workers == 0: # same-process loading - indices = next(self.sample_iter) # may raise StopIteration + if self.iter._num_workers == 0: # same-process loading + indices = next(self.iter._sampler_iter) # may raise StopIteration if len(indices) > 1: - batch = self.dataset[np.array(indices)] + batch = self.iter._dataset[np.array(indices)] else: - batch = self.collate_fn([self.dataset[i] for i in indices]) + batch = self.iter._collate_fn([self.iter._dataset[i] for i in indices]) - if self.pin_memory: + if self.iter._pin_memory: batch = _utils.pin_memory.pin_memory_batch(batch) return batch - - # check if the next sample has already been generated - if self.rcvd_idx in self.reorder_dict: - batch = self.reorder_dict.pop(self.rcvd_idx) - return self._process_next_batch(batch) - - if self.batches_outstanding == 0: - self._shutdown_workers() - raise StopIteration - - while True: - assert (not self.shutdown and self.batches_outstanding > 0) - idx, batch = self._get_batch() - self.batches_outstanding -= 1 - if idx != self.rcvd_idx: - # store out-of-order samples - self.reorder_dict[idx] = batch - continue - return self._process_next_batch(batch) + else: + return next(self.iter) diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py b/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py index 2914370033..2f52d31c2e 100644 --- a/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py +++ b/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py @@ -281,7 +281,6 @@ def __init__(self, num_workers = 0 else: num_workers = 0 - if constants.Device.CUDA in device: pin_memory = False else: @@ -294,7 +293,7 @@ def __init__(self, drop_last=True, num_workers=num_workers, pin_memory=pin_memory, - timeout=60) + timeout=0) self.f_train = LearnabilityMB(self.Nminibatch, self.D, constants.Coefficients.SLE[order], self.groups, @@ -338,7 +337,7 @@ def forward_and_backward(self, s, xsub, ysub, retain_graph=False): Completes the forward operation and computes gradients for learnability and penalty. """ f_train = self.f_train(s, xsub, ysub) - pen = self.penalty(s) + pen = self.penalty(s).unsqueeze(0).unsqueeze(0) # pylint: disable=E1102 grad_outputs = torch.tensor([[1]], dtype=torch.get_default_dtype(), device=self.device) From 1ad7dc65310c76305706eb53c6c9c7c8c3b029da Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Wed, 15 Jul 2020 17:47:52 +0800 Subject: [PATCH 2/3] updates --- .../nni/feature_engineering/gradient_selector/learnability.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py b/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py index 2f52d31c2e..d4aef29f86 100644 --- a/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py +++ b/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py @@ -281,6 +281,7 @@ def __init__(self, num_workers = 0 else: num_workers = 0 + if constants.Device.CUDA in device: pin_memory = False else: From c4e101a919d3b060cdf0442eddb7161f08f10e97 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Wed, 15 Jul 2020 18:02:26 +0800 Subject: [PATCH 3/3] updates --- .../feature_engineering/gradient_selector/learnability.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py b/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py index d4aef29f86..3a0ab4b39e 100644 --- a/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py +++ b/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py @@ -287,6 +287,11 @@ def __init__(self, else: pin_memory = False + if num_workers == 0: + timeout = 0 + else: + timeout = 60 + self.ds_train = ChunkDataLoader( PreparedData, batch_size=self.Nminibatch, @@ -294,7 +299,7 @@ def __init__(self, drop_last=True, num_workers=num_workers, pin_memory=pin_memory, - timeout=0) + timeout=timeout) self.f_train = LearnabilityMB(self.Nminibatch, self.D, constants.Coefficients.SLE[order], self.groups,