Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

take custom dataset into consideration for multi worker data loader #12093

Merged
merged 1 commit into from
Aug 10, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion python/mxnet/gluon/data/dataloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,8 @@ def _as_in_context(data, ctx):

def worker_loop(dataset, key_queue, data_queue, batchify_fn):
"""Worker loop for multiprocessing DataLoader."""
dataset._fork()
if hasattr(dataset, '_fork') and callable(dataset._fork):
dataset._fork()
while True:
idx, samples = key_queue.get()
if idx is None:
Expand Down
7 changes: 7 additions & 0 deletions tests/python/unittest/test_gluon_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,13 @@ def test_image_folder_dataset():
assert dataset.synsets == ['test_images']
assert len(dataset.items) == 16

@with_seed()
def test_list_dataset():
for num_worker in range(0, 3):
data = mx.gluon.data.DataLoader([([1,2], 0), ([3, 4], 1)], batch_size=1, num_workers=num_worker)
for d, l in data:
pass


class Dataset(gluon.data.Dataset):
def __len__(self):
Expand Down