diff --git a/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py b/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py index 16f3c08..9c514ff 100644 --- a/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py +++ b/src/cai_lab_to_nwb/zaki_2024/interfaces/miniscope_imaging_interface.py @@ -18,6 +18,39 @@ from neuroconv.utils import DeepDict, dict_deep_update +def get_miniscope_folder_path(folder_path: Union[str, Path]): + """ + Retrieve the path to the Miniscope folder within the given session folder based on metadata. + + Parameters: + ----------- + folder_path : Union[str, Path] + Path to the main session folder, which should contain a "metaData.json" file with information about the Miniscope. + + Returns: + -------- + Optional[Path] + Path to the Miniscope folder, formatted to replace any spaces in the Miniscope name with underscores. Returns `None` if the + specified folder is not a directory or if the metadata JSON is missing or misconfigured. + + Raises: + ------- + AssertionError + If the "metaData.json" file is not found in the given folder path. + """ + folder_path = Path(folder_path) + if folder_path.is_dir(): + general_metadata_json = folder_path / "metaData.json" + assert general_metadata_json.exists(), f"General metadata json not found in {folder_path}" + with open(general_metadata_json) as f: + general_metadata = json.load(f) + miniscope_name = general_metadata["miniscopes"][0] + return folder_path / miniscope_name.replace(" ", "_") + else: + print(f"No Miniscope data found at {folder_path}") + return None + + def get_recording_start_time(file_path: Union[str, Path]): """ Retrieve the recording start time from metadata in the specified folder. diff --git a/src/cai_lab_to_nwb/zaki_2024/utils/__init__.py b/src/cai_lab_to_nwb/zaki_2024/utils/__init__.py index 38262bf..7f7e2da 100644 --- a/src/cai_lab_to_nwb/zaki_2024/utils/__init__.py +++ b/src/cai_lab_to_nwb/zaki_2024/utils/__init__.py @@ -1 +1 @@ -from .cell_registration import add_cell_registration, get_global_ids_from_csv +from .edf_slicing import get_session_slicing_time_range, get_session_run_time diff --git a/src/cai_lab_to_nwb/zaki_2024/utils/edf_slicing.py b/src/cai_lab_to_nwb/zaki_2024/utils/edf_slicing.py new file mode 100644 index 0000000..3154e7e --- /dev/null +++ b/src/cai_lab_to_nwb/zaki_2024/utils/edf_slicing.py @@ -0,0 +1,63 @@ +from pathlib import Path +from typing import Union +from datetime import timedelta + +from src.cai_lab_to_nwb.zaki_2024.interfaces.miniscope_imaging_interface import ( + get_miniscope_timestamps, + get_recording_start_time, +) + + +def get_session_slicing_time_range(miniscope_metadata_json: Union[str, Path], timestamps_file_path: Union[str, Path]): + """ + Calculate the time range for EDF slicing based on session start time and Miniscope timestamps. + + Parameters: + ----------- + miniscope_metadata_json : Union[str, Path] + Path to the metadata.json file produced by Miniscope output. + + timestamps_file_path : Union[str, Path] + Path to the Miniscope timeStamps.csv file. + + Returns: + -------- + Tuple[datetime, datetime] + A tuple containing the start and stop timestamps (as datetime objects) for the EDF slicing period. The start timestamp + corresponds to the session's start time adjusted by the first Miniscope timestamp, and the stop timestamp is the session's + start time adjusted by the last Miniscope timestamp. + + """ + miniscope_metadata_json = Path(miniscope_metadata_json) + timestamps_file_path = Path(timestamps_file_path) + if miniscope_metadata_json.is_file() and timestamps_file_path.is_file(): + + session_start_time = get_recording_start_time(file_path=miniscope_metadata_json) + miniscope_timestamps = get_miniscope_timestamps(file_path=timestamps_file_path) + + start_datetime_timestamp = session_start_time + timedelta(seconds=miniscope_timestamps[0]) + stop_datetime_timestamp = session_start_time + timedelta(seconds=miniscope_timestamps[-1]) + + return start_datetime_timestamp, stop_datetime_timestamp + + +def get_session_run_time(txt_file_path: Union[str, Path]): + import re + + try: + with open(txt_file_path, "r") as file: + text = file.read() + except FileNotFoundError: + print(f"File not found at {txt_file_path}") + exit() + # Extract the Run Time line + run_time_line = re.search(r"Run Time\s*:\s*([\d:.]+)", text) + if run_time_line: + run_time_str = run_time_line.group(1) + + # Convert Run Time to seconds + h, m, s = map(float, run_time_str.split(":")) + duration = h * 3600 + m * 60 + s + return duration + else: + print("Run Time information not found.") diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py index 97dc377..45d6894 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_session.py @@ -4,79 +4,14 @@ from pathlib import Path from typing import Union -from datetime import datetime, timedelta +from datetime import datetime import pandas as pd -import json + from neuroconv.utils import load_dict_from_file, dict_deep_update from zaki_2024_nwbconverter import Zaki2024NWBConverter -from interfaces.miniscope_imaging_interface import get_miniscope_timestamps, get_recording_start_time - - -def get_miniscope_folder_path(folder_path: Union[str, Path]): - """ - Retrieve the path to the Miniscope folder within the given session folder based on metadata. - - Parameters: - ----------- - folder_path : Union[str, Path] - Path to the main session folder, which should contain a "metaData.json" file with information about the Miniscope. - - Returns: - -------- - Optional[Path] - Path to the Miniscope folder, formatted to replace any spaces in the Miniscope name with underscores. Returns `None` if the - specified folder is not a directory or if the metadata JSON is missing or misconfigured. - - Raises: - ------- - AssertionError - If the "metaData.json" file is not found in the given folder path. - """ - folder_path = Path(folder_path) - if folder_path.is_dir(): - general_metadata_json = folder_path / "metaData.json" - assert general_metadata_json.exists(), f"General metadata json not found in {folder_path}" - with open(general_metadata_json) as f: - general_metadata = json.load(f) - miniscope_name = general_metadata["miniscopes"][0] - return folder_path / miniscope_name.replace(" ", "_") - else: - print(f"No Miniscope data found at {folder_path}") - return None - - -def get_edf_slicing_time_range(miniscope_metadata_json: Union[str, Path], timestamps_file_path: Union[str, Path]): - """ - Calculate the time range for EDF slicing based on session start time and Miniscope timestamps. - - Parameters: - ----------- - miniscope_metadata_json : Union[str, Path] - Path to the metadata.json file produced by Miniscope output. - - timestamps_file_path : Union[str, Path] - Path to the Miniscope timeStamps.csv file. - - Returns: - -------- - Tuple[datetime, datetime] - A tuple containing the start and stop timestamps (as datetime objects) for the EDF slicing period. The start timestamp - corresponds to the session's start time adjusted by the first Miniscope timestamp, and the stop timestamp is the session's - start time adjusted by the last Miniscope timestamp. - - """ - miniscope_metadata_json = Path(miniscope_metadata_json) - timestamps_file_path = Path(timestamps_file_path) - if miniscope_metadata_json.is_file() and timestamps_file_path.is_file(): - - session_start_time = get_recording_start_time(file_path=miniscope_metadata_json) - miniscope_timestamps = get_miniscope_timestamps(file_path=timestamps_file_path) - - start_datetime_timestamp = session_start_time + timedelta(seconds=miniscope_timestamps[0]) - stop_datetime_timestamp = session_start_time + timedelta(seconds=miniscope_timestamps[-1]) - - return start_datetime_timestamp, stop_datetime_timestamp +from utils import get_session_slicing_time_range +from interfaces.miniscope_imaging_interface import get_miniscope_folder_path def session_to_nwb( @@ -182,7 +117,7 @@ def session_to_nwb( assert miniscope_metadata_json.exists(), f"General metadata json not found in {folder_path}" timestamps_file_path = miniscope_folder_path / "timeStamps.csv" assert timestamps_file_path.exists(), f"Miniscope timestamps file not found in {miniscope_folder_path}" - start_datetime_timestamp, stop_datetime_timestamp = get_edf_slicing_time_range( + start_datetime_timestamp, stop_datetime_timestamp = get_session_slicing_time_range( miniscope_metadata_json=miniscope_metadata_json, timestamps_file_path=timestamps_file_path ) source_data.update( diff --git a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_week_session.py b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_week_session.py index 77a4f22..e2d6e7e 100644 --- a/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_week_session.py +++ b/src/cai_lab_to_nwb/zaki_2024/zaki_2024_convert_week_session.py @@ -3,11 +3,17 @@ import time from natsort import natsorted from pathlib import Path -import warnings from typing import Union import re +import pandas as pd +from datetime import datetime +from mne.io import read_raw_edf + from neuroconv.utils import load_dict_from_file, dict_deep_update +from neuroconv.tools.nwb_helpers import configure_and_write_nwbfile +from utils import get_session_slicing_time_range, get_session_run_time +from interfaces.miniscope_imaging_interface import get_miniscope_folder_path from zaki_2024_nwbconverter import Zaki2024NWBConverter @@ -18,7 +24,6 @@ def session_to_nwb( stub_test: bool = False, verbose: bool = True, ): - if verbose: print(f"Converting week-long session") start = time.time() @@ -66,27 +71,73 @@ def session_to_nwb( conversion_options.update(dict(CellRegistration=dict(stub_test=stub_test, subject_id=subject_id))) converter = Zaki2024NWBConverter(source_data=source_data) - # Add datetime to conversion metadata = converter.get_metadata() + # Update default metadata with the editable in the corresponding yaml file + editable_metadata_path = Path(__file__).parent / "zaki_2024_metadata.yaml" + editable_metadata = load_dict_from_file(editable_metadata_path) + metadata = dict_deep_update(metadata, editable_metadata) - from mne.io import read_raw_edf + metadata["Subject"]["subject_id"] = subject_id edf_reader = read_raw_edf(input_fname=edf_file_paths[0], verbose=verbose) session_start_time = edf_reader.info["meas_date"] + metadata["NWBFile"]["session_start_time"] = session_start_time - # Update default metadata with the editable in the corresponding yaml file - editable_metadata_path = Path(__file__).parent / "zaki_2024_metadata.yaml" - editable_metadata = load_dict_from_file(editable_metadata_path) - metadata = dict_deep_update(metadata, editable_metadata) + nwbfile = converter.create_nwbfile(metadata=metadata, conversion_options=conversion_options) - metadata["Subject"]["subject_id"] = subject_id + # Add epochs table to store time range of conditioning and offline sessions + sessions_summary_file = data_dir_path / f"Ca_EEG_Experiment/{subject_id}/{subject_id}_SessionTimes.csv" + sessions_summary_df = pd.read_csv(sessions_summary_file) + + # Add columns to TimeIntervals + nwbfile.add_epoch_column(name="session_ids", description="ID of the session") + + for task, date_str, time_str in zip( + sessions_summary_df["Session"], sessions_summary_df["Date"], sessions_summary_df["Time"] + ): + session_id = subject_id + "_" + task + if "Offline" in session_id: + offline_day = session_id.split("Session")[0] + experiment_dir_path = ( + data_dir_path / "Ca_EEG_Experiment" / subject_id / (subject_id + "_Offline") / offline_day + ) + else: + experiment_dir_path = ( + data_dir_path / "Ca_EEG_Experiment" / subject_id / (subject_id + "_Sessions") / session_id + ) + try: + folder_path = experiment_dir_path / date_str / time_str + miniscope_folder_path = get_miniscope_folder_path(folder_path) + miniscope_metadata_json = folder_path / "metaData.json" + assert miniscope_metadata_json.exists(), f"General metadata json not found in {folder_path}" + timestamps_file_path = miniscope_folder_path / "timeStamps.csv" + assert timestamps_file_path.exists(), f"Miniscope timestamps file not found in {miniscope_folder_path}" + + start_datetime_timestamp, stop_datetime_timestamp = get_session_slicing_time_range( + miniscope_metadata_json=miniscope_metadata_json, timestamps_file_path=timestamps_file_path + ) + + start_time = (start_datetime_timestamp - session_start_time.replace(tzinfo=None)).total_seconds() + stop_time = (stop_datetime_timestamp - session_start_time.replace(tzinfo=None)).total_seconds() + + # Some sessions may not have imaging data, so we extract the run time from the session notes (.txt file) + # and use the data string and time string to retrieve the start datetime of the session + except: + datetime_str = date_str + " " + time_str + start_datetime_timestamp = datetime.strptime(datetime_str, "%Y_%m_%d %H_%M_%S") + + txt_file_path = experiment_dir_path / f"{session_id}.txt" + session_run_time = get_session_run_time(txt_file_path=txt_file_path) + + start_time = (start_datetime_timestamp - session_start_time.replace(tzinfo=None)).total_seconds() + stop_time = start_time + session_run_time + + nwbfile.add_epoch(start_time=start_time, stop_time=stop_time, session_ids=session_id) # Run conversion - converter.run_conversion( - metadata=metadata, nwbfile_path=nwbfile_path, conversion_options=conversion_options, overwrite=True - ) + configure_and_write_nwbfile(nwbfile=nwbfile, backend="hdf5", output_filepath=nwbfile_path) if verbose: stop_time = time.time()