enable writing directly to zipstore

note: need to enable this for non-imaris files too also, some of the remote storage features may be broken
khanlab · Jan 15, 2025 · 661b345 · 661b345
1 parent 79e61a2
commit 661b345
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 86 deletions.
diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk
@@ -278,12 +278,12 @@ def get_output_ome_zarr(acq_type):
         if config["use_zipstore"]:
             return {
                 "zarr": bids(
-                    root=work,
+                    root=root,
                     subject="{subject}",
                     datatype="micr",
                     sample="{sample}",
                     acq=f"{{acq,[a-zA-Z0-9]*{acq_type}[a-zA-Z0-9]*}}",
-                    suffix="SPIM.ome.zarr",
+                    suffix="SPIM.ome.zarr.zip",
                 )
             }
         else:
@@ -303,12 +303,12 @@ def get_output_ome_zarr(acq_type):
         if config["use_zipstore"]:
             return {
                 "zarr": bids(
-                    root=work,
+                    root=root,
                     subject="{subject}",
                     datatype="micr",
                     sample="{sample}",
                     acq=f"{{acq,[a-zA-Z0-9]*{acq_type}[a-zA-Z0-9]*}}",
-                    suffix="SPIM.ome.zarr",
+                    suffix="SPIM.ome.zarr.zip",
                 )
             }
         else:
@@ -334,7 +334,7 @@ def get_input_ome_zarr_to_nii(wildcards):
                 datatype="micr",
                 sample="{sample}",
                 acq="{acq}",
-                suffix="SPIM.ome.zarr",
+                suffix="SPIM.ome.zarr.zip",
             ).format(**wildcards)
         else:
             return bids(

diff --git a/workflow/rules/imaris.smk b/workflow/rules/imaris.smk
@@ -35,56 +35,9 @@ rule imaris_to_metadata:
     script:
         "../scripts/imaris_to_metadata.py"
 
-rule imaris_channel_to_zarr:
-    input:
-        ims=get_input_sample,
-    params:
-        channel=lambda wildcards: get_stains(wildcards).index(wildcards.stain),
-    output:
-        zarr=directory(bids(
-            root=root,
-            subject="{subject}",
-            datatype="micr",
-            sample="{sample}",
-            acq="{acq}",
-            stain="{stain}",
-            suffix="imaris.zarr",
-        )),
-    log:
-        bids(
-            root="logs",
-            subject="{subject}",
-            datatype="imaris_channel_to_zarr",
-            sample="{sample}",
-            acq="{acq}",
-            stain="{stain}",
-            suffix="log.txt",
-        ),
-    container:
-        config["containers"]["spimprep"]
-    group:
-        "preproc"
-    threads: 1
-    resources:
-        runtime=360,
-        mem_mb=1000,
-    shadow: 'minimal'
-    script:
-        "../scripts/imaris_channel_to_zarr.py"
-
-
 rule imaris_to_ome_zarr:
     input:
         ims=get_input_sample,
-        zarr=lambda wildcards: expand(bids(
-            root=root,
-            subject="{subject}",
-            datatype="micr",
-            sample="{sample}",
-            acq="{acq}",
-            stain="{stain}",
-            suffix="imaris.zarr",
-        ),stain=get_stains(wildcards),allow_missing=True),
         metadata_json=rules.prestitched_to_metadata.output.metadata_json,
     params:
         max_downsampling_layers=config["ome_zarr"]["max_downsampling_layers"],

diff --git a/workflow/rules/ome_zarr.smk b/workflow/rules/ome_zarr.smk
@@ -89,40 +89,6 @@ rule tif_stacks_to_ome_zarr:
         "../scripts/tif_stacks_to_ome_zarr.py"
 
 
-rule ome_zarr_to_zipstore:
-    """ use 7zip to create a zipstore """
-    input:
-        zarr=bids(
-            root=work,
-            subject="{subject}",
-            datatype="micr",
-            sample="{sample}",
-            acq="{acq}",
-            suffix="SPIM.ome.zarr",
-        ),
-    output:
-        zarr_zip=bids(
-            root=root,
-            subject="{subject}",
-            datatype="micr",
-            sample="{sample}",
-            acq="{acq}",
-            suffix="SPIM.ome.zarr.zip",
-        ),
-    log:
-        bids(
-            root="logs",
-            subject="{subject}",
-            datatype="micr",
-            sample="{sample}",
-            acq="{acq}",
-            suffix="log.txt",
-        ),
-    group:
-        "preproc"
-    shell:
-        "7z a -mx0 -tzip {output.zarr_zip} {input.zarr}/. &> {log}"
-
 
 rule ome_zarr_to_nii:
     input: