Merge pull request #123 from mobie/skip-add-dataset

Enable skipping addition of the source to dataset.json in add_image

mobie/image_data.py

@@ -167,7 +167,8 @@ def add_image(input_path, input_key,
               description=None,
               move_only=False,
               int_to_uint=False,
-              channel=None):
+              channel=None,
+              skip_add_to_dataset=False):
     """ Add an image source to a MoBIE dataset.
 
     Will create the dataset if it does not exist.
@@ -200,8 +201,12 @@ def add_image(input_path, input_key,
         int_to_uint [bool] - whether to convert signed to unsigned integer (default: False)
         channel [int] - the channel to load from the data.
             Currently only supported for the ome.zarr format (default: None)
+        skip_add_to_dataset [bool] - Skip adding the source to the dataset after converting the image data.
+            This should be used when calling `add_image` in parallel in order to avoid
+            writing to dataset.json in parallel, which can cause issues. In this case the source needs to be added later
+            , which can be done by calling this function again. (default: False)
     """
-    # TODO add 'setup_id' to the json schema fro bdv formats to also support it there
+    # TODO add 'setup_id' to the json schema for bdv formats to also support it there
     if channel is not None and file_format != "ome.zarr":
         raise NotImplementedError("Channel setting is currently only supported for ome.zarr")
 
@@ -244,12 +249,14 @@ def add_image(input_path, input_key,
                           int_to_uint=int_to_uint,
                           channel=channel)
 
-    metadata.add_source_to_dataset(dataset_folder, "image", image_name, image_metadata_path,
-                                   view=view, channel=channel)
-
     if transformation is not None:
         utils.update_transformation_parameter(image_metadata_path, transformation, file_format)
 
+    if skip_add_to_dataset:
+        return
+    metadata.add_source_to_dataset(dataset_folder, "image", image_name, image_metadata_path,
+                                   view=view, description=description, channel=channel)
+
 
 def main():
     description = """Add image data to MoBIE dataset.

test/test_image_data.py

@@ -285,6 +285,35 @@ def test_with_trafo_bdv_n5(self):
         # dataset_folder = os.path.join(self.root, self.dataset_name)
         # xml_path = os.path.join(self.dataset_folder, "images", "bdv-n5", f"{name}.xml")
 
+    #
+    # test skipping metadata
+    #
+
+    def test_skip_metadata(self):
+        im_name = "test-skip_metadata"
+        scales = [[2, 2, 2]]
+        mobie.add_image(self.data, None, self.root, self.dataset_name, im_name,
+                        resolution=(1, 1, 1), scale_factors=scales,
+                        chunks=(64, 64, 64), tmp_folder=self.tmp_folder,
+                        target="local", max_jobs=self.max_jobs,
+                        description="Skipping metadata.",
+                        skip_add_to_dataset=True)
+        with self.assertRaises(AssertionError):
+            self.check_data(os.path.join(self.root, self.dataset_name), im_name)
+        metadata = mobie.metadata.read_dataset_metadata(os.path.join(self.root, self.dataset_name))
+        sources = metadata["sources"]
+        self.assertEqual(sources, {})
+
+        mobie.add_image(self.data, None, self.root, self.dataset_name, im_name,
+                        resolution=(1, 1, 1), scale_factors=scales,
+                        chunks=(64, 64, 64), tmp_folder=self.tmp_folder,
+                        target="local", max_jobs=self.max_jobs,
+                        description="Add skipped metadata.",
+                        skip_add_to_dataset=False)
+
+        self.check_data(os.path.join(self.root, self.dataset_name), im_name)
+
+
     #
     # data validation
     #