From 3be3f85119d4180eef88d8945f3d1554129ac9aa Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Wed, 28 Feb 2024 19:03:20 +0800 Subject: [PATCH 1/9] remove `self._set_prop` Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- bundle/python_bundle_workflow/scripts/inference.py | 6 +----- bundle/python_bundle_workflow/scripts/train.py | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/bundle/python_bundle_workflow/scripts/inference.py b/bundle/python_bundle_workflow/scripts/inference.py index 24810a0dfb..fab321b009 100644 --- a/bundle/python_bundle_workflow/scripts/inference.py +++ b/bundle/python_bundle_workflow/scripts/inference.py @@ -63,7 +63,6 @@ def __init__(self, dataset_dir: str = "."): nib.save(n, os.path.join(dataset_dir, f"seg{i:d}.nii.gz")) self._props = {} - self._set_props = {} self.dataset_dir = dataset_dir def initialize(self): @@ -76,7 +75,7 @@ def finalize(self): pass def _set_property(self, name, property, value): - self._set_props[name] = value + self._props[name] = value def _get_property(self, name, property): """ @@ -90,9 +89,6 @@ def _get_property(self, name, property): value = None if name in self._props: value = self._props[name] - elif name in self._set_props: - value = self._set_props[name] - self._props[name] = value else: try: value = getattr(self, f"get_{name}")() diff --git a/bundle/python_bundle_workflow/scripts/train.py b/bundle/python_bundle_workflow/scripts/train.py index ccd409e969..56fc36056a 100644 --- a/bundle/python_bundle_workflow/scripts/train.py +++ b/bundle/python_bundle_workflow/scripts/train.py @@ -72,7 +72,6 @@ def __init__(self, dataset_dir: str = "."): # define buckets to store the generated properties and set properties self._props = {} - self._set_props = {} self.dataset_dir = dataset_dir # besides the predefined properties, this bundle workflow can also provide `network`, `loss`, `optimizer` @@ -91,7 +90,7 @@ def finalize(self): set_determinism(None) def _set_property(self, name, property, value): - self._set_props[name] = value + self._props[name] = value def _get_property(self, name, property): """ @@ -105,9 +104,6 @@ def _get_property(self, name, property): value = None if name in self._props: value = self._props[name] - elif name in self._set_props: - value = self._set_props[name] - self._props[name] = value else: try: value = getattr(self, f"get_{name}")() From 013a8bfef3a63bb110ffcb61bbf227a33777d180 Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:26:59 +0800 Subject: [PATCH 2/9] remove self._probs in initialize Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- bundle/python_bundle_workflow/scripts/inference.py | 2 +- bundle/python_bundle_workflow/scripts/train.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bundle/python_bundle_workflow/scripts/inference.py b/bundle/python_bundle_workflow/scripts/inference.py index fab321b009..19f9aa21ee 100644 --- a/bundle/python_bundle_workflow/scripts/inference.py +++ b/bundle/python_bundle_workflow/scripts/inference.py @@ -66,7 +66,7 @@ def __init__(self, dataset_dir: str = "."): self.dataset_dir = dataset_dir def initialize(self): - self.props = {} + pass def run(self): self.evaluator.run() diff --git a/bundle/python_bundle_workflow/scripts/train.py b/bundle/python_bundle_workflow/scripts/train.py index 56fc36056a..bc8a47afa6 100644 --- a/bundle/python_bundle_workflow/scripts/train.py +++ b/bundle/python_bundle_workflow/scripts/train.py @@ -81,7 +81,6 @@ def __init__(self, dataset_dir: str = "."): def initialize(self): set_determinism(0) - self.props = {} def run(self): self.trainer.run() From cf7448877d53908cb1949a3f9c179039902e0c2a Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Fri, 1 Mar 2024 10:44:40 +0800 Subject: [PATCH 3/9] Revert "remove `self._set_prop`" This reverts commit 3be3f85119d4180eef88d8945f3d1554129ac9aa. --- bundle/python_bundle_workflow/scripts/inference.py | 6 +++++- bundle/python_bundle_workflow/scripts/train.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/bundle/python_bundle_workflow/scripts/inference.py b/bundle/python_bundle_workflow/scripts/inference.py index 19f9aa21ee..06ce1aff5f 100644 --- a/bundle/python_bundle_workflow/scripts/inference.py +++ b/bundle/python_bundle_workflow/scripts/inference.py @@ -63,6 +63,7 @@ def __init__(self, dataset_dir: str = "."): nib.save(n, os.path.join(dataset_dir, f"seg{i:d}.nii.gz")) self._props = {} + self._set_props = {} self.dataset_dir = dataset_dir def initialize(self): @@ -75,7 +76,7 @@ def finalize(self): pass def _set_property(self, name, property, value): - self._props[name] = value + self._set_props[name] = value def _get_property(self, name, property): """ @@ -89,6 +90,9 @@ def _get_property(self, name, property): value = None if name in self._props: value = self._props[name] + elif name in self._set_props: + value = self._set_props[name] + self._props[name] = value else: try: value = getattr(self, f"get_{name}")() diff --git a/bundle/python_bundle_workflow/scripts/train.py b/bundle/python_bundle_workflow/scripts/train.py index bc8a47afa6..16fa85b324 100644 --- a/bundle/python_bundle_workflow/scripts/train.py +++ b/bundle/python_bundle_workflow/scripts/train.py @@ -72,6 +72,7 @@ def __init__(self, dataset_dir: str = "."): # define buckets to store the generated properties and set properties self._props = {} + self._set_props = {} self.dataset_dir = dataset_dir # besides the predefined properties, this bundle workflow can also provide `network`, `loss`, `optimizer` @@ -89,7 +90,7 @@ def finalize(self): set_determinism(None) def _set_property(self, name, property, value): - self._props[name] = value + self._set_props[name] = value def _get_property(self, name, property): """ @@ -103,6 +104,9 @@ def _get_property(self, name, property): value = None if name in self._props: value = self._props[name] + elif name in self._set_props: + value = self._set_props[name] + self._props[name] = value else: try: value = getattr(self, f"get_{name}")() From 84e76292e0977c3787733d88cd8ad4a6d258257f Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Fri, 1 Mar 2024 10:47:37 +0800 Subject: [PATCH 4/9] Revert "remove self._probs in initialize" This reverts commit 013a8bfef3a63bb110ffcb61bbf227a33777d180. --- bundle/python_bundle_workflow/scripts/inference.py | 2 +- bundle/python_bundle_workflow/scripts/train.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bundle/python_bundle_workflow/scripts/inference.py b/bundle/python_bundle_workflow/scripts/inference.py index 06ce1aff5f..24810a0dfb 100644 --- a/bundle/python_bundle_workflow/scripts/inference.py +++ b/bundle/python_bundle_workflow/scripts/inference.py @@ -67,7 +67,7 @@ def __init__(self, dataset_dir: str = "."): self.dataset_dir = dataset_dir def initialize(self): - pass + self.props = {} def run(self): self.evaluator.run() diff --git a/bundle/python_bundle_workflow/scripts/train.py b/bundle/python_bundle_workflow/scripts/train.py index 16fa85b324..ccd409e969 100644 --- a/bundle/python_bundle_workflow/scripts/train.py +++ b/bundle/python_bundle_workflow/scripts/train.py @@ -82,6 +82,7 @@ def __init__(self, dataset_dir: str = "."): def initialize(self): set_determinism(0) + self.props = {} def run(self): self.trainer.run() From bc610f13cd8763a9a76d4a4b5f319d60e8f3a9af Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Fri, 1 Mar 2024 11:24:55 +0800 Subject: [PATCH 5/9] address comments Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- .../scripts/inference.py | 15 ++++------ .../python_bundle_workflow/scripts/train.py | 30 +++++++++++-------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/bundle/python_bundle_workflow/scripts/inference.py b/bundle/python_bundle_workflow/scripts/inference.py index 24810a0dfb..99e092e737 100644 --- a/bundle/python_bundle_workflow/scripts/inference.py +++ b/bundle/python_bundle_workflow/scripts/inference.py @@ -38,6 +38,7 @@ ScaleIntensityd, ) from monai.utils import BundleProperty +from scripts.train import prepare_data class InferenceWorkflow(BundleWorkflow): @@ -46,7 +47,7 @@ class InferenceWorkflow(BundleWorkflow): """ - def __init__(self, dataset_dir: str = "."): + def __init__(self, dataset_dir: str = "./infer"): super().__init__(workflow="inference") print_config() # set root log level to INFO and init a evaluation logger, will be used in `StatsHandler` @@ -54,20 +55,14 @@ def __init__(self, dataset_dir: str = "."): get_logger("eval_log") # create a temporary directory and 40 random image, mask pairs - print(f"generating synthetic data to {dataset_dir} (this may take a while)") - for i in range(5): - im, seg = create_test_image_3d(128, 128, 128, num_seg_classes=1, channel_dim=-1) - n = nib.Nifti1Image(im, np.eye(4)) - nib.save(n, os.path.join(dataset_dir, f"img{i:d}.nii.gz")) - n = nib.Nifti1Image(seg, np.eye(4)) - nib.save(n, os.path.join(dataset_dir, f"seg{i:d}.nii.gz")) + prepare_data(dataset_dir=dataset_dir) self._props = {} self._set_props = {} self.dataset_dir = dataset_dir def initialize(self): - self.props = {} + self._props = {} def run(self): self.evaluator.run() @@ -112,7 +107,7 @@ def get_device(self): return torch.device("cuda" if torch.cuda.is_available() else "cpu") def get_dataset_dir(self): - return "." + return "./infer" def get_network_def(self): return UNet( diff --git a/bundle/python_bundle_workflow/scripts/train.py b/bundle/python_bundle_workflow/scripts/train.py index ccd409e969..575df30503 100644 --- a/bundle/python_bundle_workflow/scripts/train.py +++ b/bundle/python_bundle_workflow/scripts/train.py @@ -12,6 +12,7 @@ import logging import os import sys +from pathlib import Path from glob import glob import nibabel as nib @@ -48,32 +49,36 @@ from monai.utils import BundleProperty, set_determinism +def prepare_data(dataset_dir): + Path(dataset_dir).mkdir(exist_ok=True) + print(f"generating synthetic data to {dataset_dir} (this may take a while)") + for i in range(40): + im, seg = create_test_image_3d(128, 128, 128, num_seg_classes=1, channel_dim=-1) + n = nib.Nifti1Image(im, np.eye(4)) + nib.save(n, os.path.join(dataset_dir, f"img{i:d}.nii.gz")) + n = nib.Nifti1Image(seg, np.eye(4)) + nib.save(n, os.path.join(dataset_dir, f"seg{i:d}.nii.gz")) + + class TrainWorkflow(BundleWorkflow): """ Test class simulates the bundle training workflow defined by Python script directly. """ - def __init__(self, dataset_dir: str = "."): + def __init__(self, dataset_dir: str = "./train"): super().__init__(workflow="train") print_config() # set root log level to INFO and init a train logger, will be used in `StatsHandler` logging.basicConfig(stream=sys.stdout, level=logging.INFO) get_logger("train_log") - + # create a temporary directory and 40 random image, mask pairs - print(f"generating synthetic data to {dataset_dir} (this may take a while)") - for i in range(40): - im, seg = create_test_image_3d(128, 128, 128, num_seg_classes=1, channel_dim=-1) - n = nib.Nifti1Image(im, np.eye(4)) - nib.save(n, os.path.join(dataset_dir, f"img{i:d}.nii.gz")) - n = nib.Nifti1Image(seg, np.eye(4)) - nib.save(n, os.path.join(dataset_dir, f"seg{i:d}.nii.gz")) + prepare_data(dataset_dir=dataset_dir) # define buckets to store the generated properties and set properties self._props = {} self._set_props = {} - self.dataset_dir = dataset_dir # besides the predefined properties, this bundle workflow can also provide `network`, `loss`, `optimizer` self.add_property(name="network", required=True, desc="network for the training.") @@ -82,7 +87,7 @@ def __init__(self, dataset_dir: str = "."): def initialize(self): set_determinism(0) - self.props = {} + self._props = {} def run(self): self.trainer.run() @@ -91,6 +96,7 @@ def finalize(self): set_determinism(None) def _set_property(self, name, property, value): + # stores user-reset initialized objects that should not be re-initialized. self._set_props[name] = value def _get_property(self, name, property): @@ -127,7 +133,7 @@ def get_device(self): return torch.device("cuda" if torch.cuda.is_available() else "cpu") def get_dataset_dir(self): - return "." + return "./train" def get_network(self): return UNet( From 5bc6ffe53993176949223a55fe036132a1a259ea Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 1 Mar 2024 03:25:55 +0000 Subject: [PATCH 6/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- bundle/python_bundle_workflow/scripts/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/python_bundle_workflow/scripts/train.py b/bundle/python_bundle_workflow/scripts/train.py index 575df30503..7eb19cb177 100644 --- a/bundle/python_bundle_workflow/scripts/train.py +++ b/bundle/python_bundle_workflow/scripts/train.py @@ -72,7 +72,7 @@ def __init__(self, dataset_dir: str = "./train"): # set root log level to INFO and init a train logger, will be used in `StatsHandler` logging.basicConfig(stream=sys.stdout, level=logging.INFO) get_logger("train_log") - + # create a temporary directory and 40 random image, mask pairs prepare_data(dataset_dir=dataset_dir) From e0eb904268bc1fac2977588d41bbb7888b2f5318 Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Fri, 1 Mar 2024 11:26:21 +0800 Subject: [PATCH 7/9] DCO Remediation Commit for YunLiu <55491388+KumoLiu@users.noreply.github.com> I, YunLiu <55491388+KumoLiu@users.noreply.github.com>, hereby add my Signed-off-by to this commit: cf7448877d53908cb1949a3f9c179039902e0c2a I, YunLiu <55491388+KumoLiu@users.noreply.github.com>, hereby add my Signed-off-by to this commit: 84e76292e0977c3787733d88cd8ad4a6d258257f Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- bundle/python_bundle_workflow/scripts/inference.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/python_bundle_workflow/scripts/inference.py b/bundle/python_bundle_workflow/scripts/inference.py index 99e092e737..93ddad7847 100644 --- a/bundle/python_bundle_workflow/scripts/inference.py +++ b/bundle/python_bundle_workflow/scripts/inference.py @@ -71,6 +71,7 @@ def finalize(self): pass def _set_property(self, name, property, value): + # stores user-reset initialized objects that should not be re-initialized. self._set_props[name] = value def _get_property(self, name, property): From e94cf15a35e68afa9e32ce46b3443551e4f493ed Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:51:37 +0800 Subject: [PATCH 8/9] address comments Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- bundle/python_bundle_workflow/scripts/inference.py | 6 +++--- bundle/python_bundle_workflow/scripts/train.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bundle/python_bundle_workflow/scripts/inference.py b/bundle/python_bundle_workflow/scripts/inference.py index 93ddad7847..9d05c1218b 100644 --- a/bundle/python_bundle_workflow/scripts/inference.py +++ b/bundle/python_bundle_workflow/scripts/inference.py @@ -84,11 +84,11 @@ def _get_property(self, name, property): """ value = None - if name in self._props: - value = self._props[name] - elif name in self._set_props: + if name in self._set_props: value = self._set_props[name] self._props[name] = value + elif name in self._props: + value = self._props[name] else: try: value = getattr(self, f"get_{name}")() diff --git a/bundle/python_bundle_workflow/scripts/train.py b/bundle/python_bundle_workflow/scripts/train.py index 7eb19cb177..d1282daf6e 100644 --- a/bundle/python_bundle_workflow/scripts/train.py +++ b/bundle/python_bundle_workflow/scripts/train.py @@ -109,11 +109,11 @@ def _get_property(self, name, property): """ value = None - if name in self._props: - value = self._props[name] - elif name in self._set_props: + if name in self._set_props: value = self._set_props[name] self._props[name] = value + elif name in self._props: + value = self._props[name] else: try: value = getattr(self, f"get_{name}")() From 5a4d80c17c1d93ddf9a3142e0a68171893c4759b Mon Sep 17 00:00:00 2001 From: YunLiu <55491388+KumoLiu@users.noreply.github.com> Date: Mon, 4 Mar 2024 15:06:40 +0800 Subject: [PATCH 9/9] address comments Signed-off-by: YunLiu <55491388+KumoLiu@users.noreply.github.com> --- bundle/python_bundle_workflow/scripts/inference.py | 2 +- bundle/python_bundle_workflow/scripts/train.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle/python_bundle_workflow/scripts/inference.py b/bundle/python_bundle_workflow/scripts/inference.py index 9d05c1218b..a98b47bb64 100644 --- a/bundle/python_bundle_workflow/scripts/inference.py +++ b/bundle/python_bundle_workflow/scripts/inference.py @@ -108,7 +108,7 @@ def get_device(self): return torch.device("cuda" if torch.cuda.is_available() else "cpu") def get_dataset_dir(self): - return "./infer" + return self.dataset_dir def get_network_def(self): return UNet( diff --git a/bundle/python_bundle_workflow/scripts/train.py b/bundle/python_bundle_workflow/scripts/train.py index d1282daf6e..29a53b46a1 100644 --- a/bundle/python_bundle_workflow/scripts/train.py +++ b/bundle/python_bundle_workflow/scripts/train.py @@ -79,6 +79,7 @@ def __init__(self, dataset_dir: str = "./train"): # define buckets to store the generated properties and set properties self._props = {} self._set_props = {} + self.dataset_dir = dataset_dir # besides the predefined properties, this bundle workflow can also provide `network`, `loss`, `optimizer` self.add_property(name="network", required=True, desc="network for the training.") @@ -133,7 +134,7 @@ def get_device(self): return torch.device("cuda" if torch.cuda.is_available() else "cpu") def get_dataset_dir(self): - return "./train" + return self.dataset_dir def get_network(self): return UNet(