Skip to content

Commit

Permalink
Merge pull request #632 from catalystneuro/change_roiresponseseries_s…
Browse files Browse the repository at this point in the history 
…chema_from_array_to_object_type

Change `Fluorescence`/`DFOverF` metadata schema for RoiResponseSeries from `array` type to `object`
  • Loading branch information
@CodyCBakerPhD
CodyCBakerPhD authored Nov 11, 2023
2 parents 1bb6e14 + cf84ab7 commit 5333be8
Show file tree
Hide file tree
Showing 5 changed files with 122 additions and 65 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

### Features
* Added Pydantic data models of `BackendConfiguration` for both HDF5 and Zarr datasets (container/mapper of all the `DatasetConfiguration`s for a particular file). [PR #568](https://github.com/catalystneuro/neuroconv/pull/568)
* Changed the metadata schema for `Fluorescence` and `DfOverF` where the traces metadata can be provided as a dict instead of a list of dicts.
The name of the plane segmentation is used to determine which traces to add to the `Fluorescence` and `DfOverF` containers. [PR #632](https://github.com/catalystneuro/neuroconv/pull/632)

### Fixes
* Fixed GenericDataChunkIterator (in hdmf.py) in the case where the number of dimensions is 1 and the size in bytes is greater than the threshold of 1 GB. [PR #638](https://github.com/catalystneuro/neuroconv/pull/638)
Expand Down
22 changes: 16 additions & 6 deletions src/neuroconv/datainterfaces/ophys/basesegmentationextractorinterface.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,23 @@ def get_metadata_schema(self) -> dict:
metadata_schema["properties"]["Ophys"]["properties"]["ImagingPlane"].update(type="array")
metadata_schema["properties"]["Ophys"]["properties"]["TwoPhotonSeries"].update(type="array")

metadata_schema["properties"]["Ophys"]["properties"]["Fluorescence"]["properties"]["roi_response_series"][
"items"
]["required"] = list()
metadata_schema["properties"]["Ophys"]["properties"]["ImageSegmentation"]["additionalProperties"] = True
metadata_schema["properties"]["Ophys"]["properties"]["Fluorescence"]["properties"]["roi_response_series"].pop(
"maxItems"
metadata_schema["properties"]["Ophys"]["properties"]["Fluorescence"].update(required=["name"])
metadata_schema["properties"]["Ophys"]["properties"]["Fluorescence"].pop("additionalProperties")
roi_response_series_schema = metadata_schema["properties"]["Ophys"]["properties"]["Fluorescence"][
"properties"
].pop("roi_response_series")

roi_response_series_schema.pop("maxItems")
roi_response_series_schema["items"].update(required=list())
roi_response_series_per_plane_schema = dict(
type="object", patternProperties={"^[a-zA-Z0-9]+$": roi_response_series_schema}
)
metadata_schema["properties"]["Ophys"]["properties"]["Fluorescence"]["properties"].update(
patternProperties={"^[a-zA-Z0-9]+$": roi_response_series_per_plane_schema}
)

metadata_schema["properties"]["Ophys"]["properties"]["ImageSegmentation"]["additionalProperties"] = True

metadata_schema["properties"]["Ophys"]["properties"]["DfOverF"] = metadata_schema["properties"]["Ophys"][
"properties"
]["Fluorescence"]
Expand Down
30 changes: 22 additions & 8 deletions src/neuroconv/schemas/metadata_schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,10 +166,17 @@
"required": ["name"],
"properties": {
"name": {"type": "string", "default": "DfOverF"},
"roi_response_series": {
"type": "array",
"title": "ROI response series",
"items": {"$ref": "#/definitions/RoiResponseSeries"}
"patternProperties": {
"^[a-zA-Z0-9]+$": {
"type": "object",
"patternProperties":{
"^[a-zA-Z0-9]+$": {
"type": "object",
"title": "ROI response series",
"properties": {"$ref": "#/definitions/RoiResponseSeries"}
}
}
}
}
}
},
Expand All @@ -178,10 +185,17 @@
"required": ["name"],
"properties": {
"name": {"type": "string", "default": "Fluorescence"},
"roi_response_series": {
"type": "array",
"title": "ROI response series",
"items": {"$ref": "#/definitions/RoiResponseSeries"}
"patternProperties": {
"^[a-zA-Z0-9]+$": {
"type": "object",
"patternProperties":{
"^[a-zA-Z0-9]+$": {
"type": "object",
"title": "ROI response series",
"properties": {"$ref": "#/definitions/RoiResponseSeries"}
}
}
}
}
}
},
Expand Down
43 changes: 23 additions & 20 deletions src/neuroconv/tools/roiextractors/roiextractors.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,9 @@ def get_default_segmentation_metadata() -> DeepDict:

default_fluorescence = dict(
name="Fluorescence",
roi_response_series=[default_fluorescence_roi_response_series],
PlaneSegmentation=dict(
raw=default_fluorescence_roi_response_series,
),
)

default_dff_roi_response_series = dict(
Expand All @@ -95,7 +97,9 @@ def get_default_segmentation_metadata() -> DeepDict:

default_df_over_f = dict(
name="DfOverF",
roi_response_series=[default_dff_roi_response_series],
PlaneSegmentation=dict(
dff=default_dff_roi_response_series,
),
)

default_image_segmentation = dict(
Expand Down Expand Up @@ -650,13 +654,16 @@ def get_nwb_segmentation_metadata(sgmextractor: SegmentationExtractor) -> dict:
description=f"{ch_name} description",
)
)

plane_segmentation_name = metadata["Ophys"]["ImageSegmentation"]["plane_segmentations"][0]["name"]
for trace_name, trace_data in sgmextractor.get_traces_dict().items():
# raw traces have already default name ("RoiResponseSeries")
if trace_name in ["raw", "dff"]:
continue
if trace_data is not None and len(trace_data.shape) != 0:
metadata["Ophys"]["Fluorescence"]["roi_response_series"].append(
dict(
name=trace_name.capitalize(),
description=f"description of {trace_name} traces",
)
metadata["Ophys"]["Fluorescence"][plane_segmentation_name][trace_name] = dict(
name=trace_name.capitalize(),
description=f"description of {trace_name} traces",
)

return metadata
Expand Down Expand Up @@ -945,23 +952,19 @@ def add_fluorescence_traces(
for trace_name, trace in traces_to_add.items():
# Decide which data interface to use based on the trace name
data_interface = trace_to_data_interface[trace_name]
# Extract the response series metadata
# The name of the roi_response_series is "RoiResponseSeries" for raw and df/F traces,
# otherwise it is capitalized trace_name.
trace_name = "RoiResponseSeries" if trace_name in ["raw", "dff"] else trace_name.capitalize()

if trace_name in data_interface.roi_response_series:
continue

data_interface_metadata = df_over_f_metadata if isinstance(data_interface, DfOverF) else fluorescence_metadata
response_series_metadata = data_interface_metadata["roi_response_series"]
trace_metadata = next(
(trace_metadata for trace_metadata in response_series_metadata if trace_name == trace_metadata["name"]),
None,
# Extract the response series metadata
# the name of the trace is retrieved from the metadata, no need to override it here
# trace_name = "RoiResponseSeries" if trace_name in ["raw", "dff"] else trace_name.capitalize()
assert plane_segmentation_name in data_interface_metadata, (
f"Plane segmentation '{plane_segmentation_name}' not found in " f"{data_interface_metadata} metadata."
)
trace_metadata = data_interface_metadata[plane_segmentation_name][trace_name]
if trace_metadata is None:
raise ValueError(f"Metadata for '{trace_name}' trace not found in {response_series_metadata}.")
raise ValueError(f"Metadata for '{trace_name}' trace not found in {data_interface_metadata}.")

if trace_metadata["name"] in data_interface.roi_response_series:
continue
# Pop the rate from the metadata if irregular time series
if "timestamps" in roi_response_series_kwargs and "rate" in trace_metadata:
trace_metadata.pop("rate")
Expand Down
90 changes: 59 additions & 31 deletions tests/test_ophys/test_tools_roiextractors.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -822,19 +822,21 @@ def setUp(self):

fluorescence_metadata = dict(
Fluorescence=dict(
name=self.fluorescence_name,
roi_response_series=[
self.raw_roi_response_series_metadata,
self.deconvolved_roi_response_series_metadata,
self.neuropil_roi_response_series_metadata,
],
PlaneSegmentation=dict(
name=self.fluorescence_name,
raw=self.raw_roi_response_series_metadata,
deconvolved=self.deconvolved_roi_response_series_metadata,
neuropil=self.neuropil_roi_response_series_metadata,
)
)
)

dff_metadata = dict(
DfOverF=dict(
name=self.df_over_f_name,
roi_response_series=[self.dff_roi_response_series_metadata],
PlaneSegmentation=dict(
name=self.df_over_f_name,
dff=self.dff_roi_response_series_metadata,
)
)
)

Expand Down Expand Up @@ -1165,8 +1167,8 @@ def test_add_fluorescence_traces_regular_timestamps_with_metadata(self):
segmentation_extractor.set_times(times)

metadata = deepcopy(self.metadata)
metadata["Ophys"]["Fluorescence"]["roi_response_series"][0].update(rate=1.23)
metadata["Ophys"]["DfOverF"]["roi_response_series"][0].update(rate=1.23)
metadata["Ophys"]["Fluorescence"]["PlaneSegmentation"]["raw"].update(rate=1.23)
metadata["Ophys"]["DfOverF"]["PlaneSegmentation"]["dff"].update(rate=1.23)

add_fluorescence_traces(
segmentation_extractor=segmentation_extractor,
Expand All @@ -1193,7 +1195,7 @@ def test_add_fluorescence_traces_irregular_timestamps_with_metadata(self):
segmentation_extractor.set_times(times)

metadata = deepcopy(self.metadata)
metadata["Ophys"]["Fluorescence"]["roi_response_series"][0].update(rate=1.23)
metadata["Ophys"]["Fluorescence"]["PlaneSegmentation"]["raw"].update(rate=1.23)

add_fluorescence_traces(
segmentation_extractor=segmentation_extractor,
Expand All @@ -1214,6 +1216,10 @@ def test_add_fluorescence_traces_with_plane_segmentation_name_specified(self):
metadata = dict_deep_update(metadata, self.metadata)

metadata["Ophys"]["ImageSegmentation"]["plane_segmentations"][0].update(name=plane_segmentation_name)
metadata["Ophys"]["Fluorescence"][plane_segmentation_name] = metadata["Ophys"]["Fluorescence"].pop(
"PlaneSegmentation"
)
metadata["Ophys"]["DfOverF"][plane_segmentation_name] = metadata["Ophys"]["DfOverF"].pop("PlaneSegmentation")

add_fluorescence_traces(
segmentation_extractor=self.segmentation_extractor,
Expand Down Expand Up @@ -1247,8 +1253,8 @@ def setUpClass(cls):
name=cls.plane_segmentation_first_plane_name
)

cls.fluorescence_name = "FluorescenceFirstPlane"
cls.df_over_f_name = "DfOverFFirstPlane"
cls.fluorescence_name = "Fluorescence"
cls.df_over_f_name = "DfOverF"

cls.raw_roi_response_series_metadata = dict(
name="RoiResponseSeries",
Expand All @@ -1272,15 +1278,22 @@ def setUpClass(cls):
)

cls.metadata["Ophys"]["Fluorescence"].update(
name=cls.fluorescence_name,
roi_response_series=[
cls.raw_roi_response_series_metadata,
cls.deconvolved_roi_response_series_metadata,
cls.neuropil_roi_response_series_metadata,
],
{
cls.plane_segmentation_first_plane_name: dict(
name=cls.fluorescence_name,
raw=cls.raw_roi_response_series_metadata,
deconvolved=cls.deconvolved_roi_response_series_metadata,
neuropil=cls.neuropil_roi_response_series_metadata,
)
}
)
cls.metadata["Ophys"]["DfOverF"].update(
name=cls.df_over_f_name, roi_response_series=[cls.dff_roi_response_series_metadata]
{
cls.plane_segmentation_first_plane_name: dict(
name=cls.df_over_f_name,
dff=cls.dff_roi_response_series_metadata,
)
}
)

def setUp(self):
Expand Down Expand Up @@ -1322,10 +1335,21 @@ def test_add_fluorescence_traces_for_two_plane_segmentations(self):
imaging_plane=second_imaging_plane_name,
)

fluorescence_second_plane_name = "FluorescenceSecondPlane"
metadata["Ophys"]["Fluorescence"].update(name=fluorescence_second_plane_name)
df_over_f_second_plane_name = "DfOverFSecondPlane"
metadata["Ophys"]["DfOverF"].update(name=df_over_f_second_plane_name)
metadata["Ophys"]["Fluorescence"][second_plane_segmentation_name] = deepcopy(
metadata["Ophys"]["Fluorescence"][self.plane_segmentation_first_plane_name]
)
metadata["Ophys"]["DfOverF"][second_plane_segmentation_name] = deepcopy(
metadata["Ophys"]["DfOverF"][self.plane_segmentation_first_plane_name]
)

metadata["Ophys"]["Fluorescence"][second_plane_segmentation_name]["raw"].update(
name="RoiResponseSeriesSecondPlane"
)
metadata["Ophys"]["Fluorescence"][second_plane_segmentation_name]["deconvolved"].update(
name="DeconvolvedSecondPlane"
)
metadata["Ophys"]["Fluorescence"][second_plane_segmentation_name]["neuropil"].update(name="NeuropilSecondPlane")
metadata["Ophys"]["DfOverF"][second_plane_segmentation_name]["dff"].update(name="RoiResponseSeriesSecondPlane")

add_fluorescence_traces(
segmentation_extractor=self.segmentation_extractor_second_plane,
Expand All @@ -1344,16 +1368,20 @@ def test_add_fluorescence_traces_for_two_plane_segmentations(self):
self.assertEqual(second_plane_segmentation.name, second_plane_segmentation_name)
self.assertEqual(second_plane_segmentation.description, "second plane segmentation description")

fluorescence_first_plane = ophys.get(self.fluorescence_name)
self.assertEqual(fluorescence_first_plane.name, self.fluorescence_name)
self.assertEqual(len(fluorescence_first_plane.roi_response_series), 3)
fluorescence = ophys.get(self.fluorescence_name)
self.assertEqual(fluorescence.name, self.fluorescence_name)
self.assertEqual(len(fluorescence.roi_response_series), 6)

fluorescence_second_plane = ophys.get(fluorescence_second_plane_name)
self.assertEqual(fluorescence_second_plane.name, fluorescence_second_plane_name)
self.assertEqual(len(fluorescence_second_plane.roi_response_series), 3)
df_over_f = ophys.get(self.df_over_f_name)
self.assertEqual(df_over_f.name, self.df_over_f_name)
self.assertEqual(len(df_over_f.roi_response_series), 2)

self.assertEqual(
fluorescence_second_plane.roi_response_series["RoiResponseSeries"].data.maxshape,
fluorescence.roi_response_series["RoiResponseSeriesSecondPlane"].data.maxshape,
(self.num_frames, self.num_rois_second_plane),
)
self.assertEqual(
df_over_f.roi_response_series["RoiResponseSeriesSecondPlane"].data.maxshape,
(self.num_frames, self.num_rois_second_plane),
)

Expand Down

0 comments on commit 5333be8

Please sign in to comment.