Merge pull request #9 from catalystneuro/add_miniscope_utils

Add miniscope utils

README.md

@@ -4,14 +4,37 @@ This is a Neurodata Extension (NDX) for Neurodata Without Borders (NWB) 2.0 for
 
 [![PyPI version](https://badge.fury.io/py/ndx-miniscope.svg)](https://badge.fury.io/py/ndx-miniscope)
 
-The Miniscope acquisition software generally outputs the following files:
+`Miniscope` extends the `Device` core NWB neurodata_type by including additional metadata for the Miniscope.
+Depending on the version of the acquisition software the data structure can be quite different.
+
+## Miniscope V4 format
+The data recorded by the software is saved in a folder structure similar to this:
+
+        C6-J588_Disc5/ (main folder)
+        ├── 15_03_28/ (subfolder corresponding to the recording time)
+        │   ├── Miniscope/ (subfolder containing the microscope video stream)
+        │   │   ├── 0.avi (microscope video)
+        │   │   ├── metaData.json (metadata for the microscope device)
+        │   │   └── timeStamps.csv (timing of this video stream)
+        │   ├── BehavCam_2/ (subfolder containing the behavioral video stream)
+        │   │   ├── 0.avi (bevavioral video)
+        │   │   ├── metaData.json (metadata for the behavioral camera)
+        │   │   └── timeStamps.csv (timing of this video stream)
+        │   └── metaData.json (metadata for the recording, such as the start time)
+        ├── 15_06_28/
+        │   ├── Miniscope/
+        │   ├── BehavCam_2/
+        │   └── metaData.json
+        └── 15_12_28/
+
+## Miniscope V3 format
+The Miniscope V3 acquisition software generally outputs the following files:
 
 * msCam[##].avi
 * behavCam[##].avi
 * timestamp.dat
 * settings_and_notes.dat
 
-This repo provides an extension to the `Device` core NWB neurodata_type called `Miniscope` which contains fields for the data in `settings_and_notes.dat`. The following code demonstrates how to use this extension to properly convert Miniscope acquisition data into NWB by creating external links, which does not require the video data to be copied into the NWB file.
 
 ## python
 ### Installation
@@ -28,65 +51,94 @@ cd ndx-miniscope
 pip install -e .
 ```
 
-### Usage
+The following code demonstrates the usage of this extension to convert Miniscope acquisition data into NWB.
 
+### Usage
 
 ```python
-import os
-from ndx_miniscope import read_settings, read_notes, read_miniscope_timestamps, get_starting_frames
-from pynwb import NWBFile, NWBHDF5IO
 from datetime import datetime
 from dateutil.tz import tzlocal
+import glob
+import os
+from pynwb import NWBFile, NWBHDF5IO
 from pynwb.image import ImageSeries
 from natsort import natsorted
-from glob import glob
 
+from ndx_miniscope.utils import (
+    add_miniscope_device,
+    get_starting_frames,
+    get_timestamps,
+    read_miniscope_config,
+    read_notes,
+)
 
-data_dir = 'path/to/data_dir'
+# The main folder that contains subfolders with the Miniscope data
+folder_path = "C6-J588_Disc5/"
 
+# Create the NWBFile
 session_start_time = datetime(2017, 4, 15, 12, tzinfo=tzlocal())
+nwbfile = NWBFile(
+    session_description="session_description",
+    identifier="identifier",
+    session_start_time=session_start_time,
+)
 
-nwb = NWBFile('session_description', 'identifier', session_start_time)
+# Load the miscroscope settings
+miniscope_folder_path = "C6-J588_Disc5/15_03_28/Miniscope/"
+miniscope_metadata = read_miniscope_config(folder_path=miniscope_folder_path)
+# Create the Miniscope device with the microscope metadata and add it to NWB
+add_miniscope_device(nwbfile=nwbfile, device_metadata=miniscope_metadata)
 
-miniscope = read_settings(data_dir)
-nwb.add_device(miniscope)
+# Load the behavioral camera settings
+behavcam_folder_path = "C6-J588_Disc5/15_03_28/BehavCam_2/"
+behavcam_metadata = read_miniscope_config(folder_path=behavcam_folder_path)
+# Create the Miniscope device with the behavioral camera metadata and add it to NWB
+add_miniscope_device(nwbfile=nwbfile, device_metadata=behavcam_metadata)
 
-annotations = read_notes(data_dir)
-if annotations is not None:
-    nwb.add_acquisition(annotations)
+# Loading the timestamps
+behavcam_timestamps = get_timestamps(folder_path=folder_path, file_pattern="BehavCam*/timeStamps.csv")
+# Load the starting frames of the video files
+# Note this function requires to have `cv2` installed
+starting_frames = get_starting_frames(folder_path=folder_path, video_file_pattern="*/BehavCam*/*.avi")
 
-ms_files = natsorted(glob(os.path.join(data_dir, 'msCam*.avi')))
-behav_files = natsorted(glob(os.path.join(data_dir, 'behavCam*.avi')))
 
+# Legacy usage for Miniscope V3
 
-nwb.add_acquisition(
+ms_files = natsorted(glob(os.path.join(folder_path, 'msCam*.avi')))
+nwbfile.add_acquisition(
     ImageSeries(
-        name='OnePhotonSeries',
+        name='OnePhotonSeries',  # this is not recommended since pynwb has native OnePhotonSeries
         format='external',
         external_file=[os.path.split(x)[1] for x in ms_files],
-        timestamps=read_miniscope_timestamps(data_dir),
-        starting_frame=get_starting_frames(ms_files),
+        timestamps=get_timestamps(folder_path=folder_path, cam_num=1),
+        starting_frame=get_starting_frames(folder_path=folder_path, video_file_pattern="msCam*.avi"),
     )
 )
 
-nwb.add_acquisition(
+behav_files = natsorted(glob(os.path.join(folder_path, 'behavCam*.avi')))
+nwbfile.add_acquisition(
     ImageSeries(
         name='behaviorCam',
         format='external',
         external_file=[os.path.split(x)[1] for x in behav_files],
-        timestamps=read_miniscope_timestamps(data_dir, cam_num=2),
-        starting_frame=get_starting_frames(behav_files),
+        timestamps=get_timestamps(folder_path=folder_path, cam_num=2),
+        starting_frame=get_starting_frames(folder_path=folder_path, video_file_pattern="behavCam*.avi"),
     )
 )
 
+annotations = read_notes(folder_path=folder_path)
+if annotations is not None:
+    nwbfile.add_acquisition(annotations)
+
 
-save_path = os.path.join(data_dir, 'test_out.nwb')
-with NWBHDF5IO(save_path, 'w') as io:
-    io.write(nwb)
+save_path = os.path.join(folder_path, "test_out.nwb")
+with NWBHDF5IO(save_path, "w") as io:
+    io.write(nwbfile)
 
 # test read
-with NWBHDF5IO(save_path, 'r') as io:
-    nwb = io.read()
+with NWBHDF5IO(save_path, "r") as io:
+    nwbfile_in = io.read()
+
 ```
 
 

requirements.txt

@@ -1,2 +1,3 @@
 pynwb
 nwb_docutils
+natsort>=8.3.1

setup.py

@@ -16,7 +16,7 @@
 
 setup_args = {
     "name": "ndx-miniscope",
-    "version": "0.4.0",
+    "version": "0.5.0",
     "description": "Represent metadata for Miniscope acquisition system.",
     "long_description": long_description,
     "long_description_content_type": "text/markdown",

spec/ndx-miniscope.namespace.yaml

@@ -8,4 +8,4 @@ namespaces:
     neurodata_types:
     - Device
   - source: ndx-miniscope.extensions.yaml
-  version: 0.4.0
+  version: 0.5.0

src/pynwb/ndx_miniscope/__init__.py

@@ -1 +1,12 @@
-from .miniscope import *
+import os
+
+from pynwb import get_class, load_namespaces
+
+name = "ndx-miniscope"
+
+here = os.path.abspath(os.path.dirname(__file__))
+ndx_miniscope_spec_path = os.path.join(here, "spec", name + ".namespace.yaml")
+
+load_namespaces(ndx_miniscope_spec_path)
+
+Miniscope = get_class("Miniscope", name)

src/pynwb/ndx_miniscope/utils/__init__.py

@@ -0,0 +1,9 @@
+from .notes import read_notes
+from .nwb import add_miniscope_device, add_miniscope_image_series
+from .settings import read_miniscope_config
+from .timestamps import (
+    get_recording_start_times,
+    get_timestamps,
+    read_miniscope_timestamps,
+)
+from .video import get_starting_frames

src/pynwb/ndx_miniscope/utils/notes.py

@@ -0,0 +1,35 @@
+import os
+
+import pandas as pd
+from packaging import version
+from pynwb.misc import AnnotationSeries
+
+from ..utils.settings import get_miniscope_version
+
+
+def read_notes(folder_path: str):
+    """Reads the notes from the settings_and_notes.dat file and creates a pynwb.misc.AnnotationSeries
+
+    Parameters
+    ----------
+    folder_path: str
+        data dir containing settings_and_notes.dat
+
+    Returns
+    -------
+    None or pynwb.misc.AnnotationSeries
+
+    """
+    miniscope_version = get_miniscope_version(folder_path=folder_path)
+    if miniscope_version == version.Version("4"):
+        raise NotImplementedError("This function is not supported for Miniscope V4 format.")
+
+    fpath = os.path.join(folder_path, "settings_and_notes.dat")
+    df = pd.read_csv(fpath, skiprows=3, delimiter="\t")
+    if len(df):
+        return AnnotationSeries(
+            name="notes",
+            data=df["Note"].values,
+            timestamps=df["elapsedTime"].values / 1000,
+            description="read from miniscope settings_and_notes.dat file",
+        )