diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache/tables/sessions_table.py b/allensdk/brain_observatory/behavior/behavior_project_cache/tables/sessions_table.py index e3e5a37a8..6700a6c7b 100644 --- a/allensdk/brain_observatory/behavior/behavior_project_cache/tables/sessions_table.py +++ b/allensdk/brain_observatory/behavior/behavior_project_cache/tables/sessions_table.py @@ -1,41 +1,42 @@ import re -from typing import Optional, List, Dict +from typing import Dict, List, Optional import pandas as pd - -from allensdk.brain_observatory.behavior.behavior_project_cache.tables \ - .ophys_sessions_table import \ - BehaviorOphysSessionsTable -from allensdk.brain_observatory.behavior.behavior_project_cache.tables \ - .util.prior_exposure_processing import \ - get_prior_exposures_to_session_type, get_prior_exposures_to_image_set, \ - get_prior_exposures_to_omissions -from allensdk.brain_observatory.behavior.behavior_project_cache.tables \ - .project_table import \ - ProjectTable -from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io import BehaviorProjectLimsApi # noqa: E501 - +from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io import ( # noqa: E501 + BehaviorProjectLimsApi, +) +from allensdk.brain_observatory.behavior.behavior_project_cache.tables.ophys_mixin import ( # noqa: E501 + OphysMixin, +) +from allensdk.brain_observatory.behavior.behavior_project_cache.tables.ophys_sessions_table import ( # noqa: E501 + BehaviorOphysSessionsTable, +) +from allensdk.brain_observatory.behavior.behavior_project_cache.tables.project_table import ( # noqa: E501 + ProjectTable, +) +from allensdk.brain_observatory.behavior.behavior_project_cache.tables.util.prior_exposure_processing import ( # noqa: E501 + get_prior_exposures_to_image_set, + get_prior_exposures_to_omissions, + get_prior_exposures_to_session_type, +) from allensdk.brain_observatory.behavior.data_files import BehaviorStimulusFile from allensdk.brain_observatory.behavior.data_objects import StimulusTimestamps from allensdk.brain_observatory.behavior.data_objects.licks import Licks - -from allensdk.brain_observatory.behavior.data_objects.metadata\ - .subject_metadata.full_genotype import \ - FullGenotype - -from allensdk.brain_observatory.behavior.data_objects.metadata\ - .subject_metadata.reporter_line import \ - ReporterLine +from allensdk.brain_observatory.behavior.data_objects.metadata.subject_metadata.full_genotype import ( # noqa: E501 + FullGenotype, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.subject_metadata.reporter_line import ( # noqa: E501 + ReporterLine, +) from allensdk.brain_observatory.behavior.data_objects.rewards import Rewards -from allensdk.brain_observatory.behavior.data_objects.trials.trials import \ - Trials +from allensdk.brain_observatory.behavior.data_objects.trials.trials import ( + Trials, +) from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP from allensdk.internal.api import db_connection_creator -from allensdk.internal.brain_observatory.util.multi_session_utils import \ - multiprocessing_helper -from allensdk.brain_observatory.behavior.behavior_project_cache.tables\ - .ophys_mixin import \ - OphysMixin +from allensdk.internal.brain_observatory.util.multi_session_utils import ( + multiprocessing_helper, +) class SessionsTable(ProjectTable, OphysMixin): @@ -43,11 +44,12 @@ class SessionsTable(ProjectTable, OphysMixin): at the session level""" def __init__( - self, df: pd.DataFrame, - fetch_api: BehaviorProjectLimsApi, - suppress: Optional[List[str]] = None, - ophys_session_table: Optional[BehaviorOphysSessionsTable] = None, - include_trial_metrics: bool = False + self, + df: pd.DataFrame, + fetch_api: BehaviorProjectLimsApi, + suppress: Optional[List[str]] = None, + ophys_session_table: Optional[BehaviorOphysSessionsTable] = None, + include_trial_metrics: bool = False, ): """ Parameters @@ -73,24 +75,31 @@ def __init__( def postprocess_additional(self): # Add subject metadata - self._df['reporter_line'] = self._df['reporter_line'].apply( - ReporterLine.parse) - self._df['cre_line'] = self._df['full_genotype'].apply( - lambda x: FullGenotype(x).parse_cre_line()) - self._df['indicator'] = self._df['reporter_line'].apply( - lambda x: ReporterLine(x).parse_indicator()) + self._df["reporter_line"] = self._df["reporter_line"].apply( + ReporterLine.parse + ) + self._df["cre_line"] = self._df["full_genotype"].apply( + lambda x: FullGenotype(x).parse_cre_line() + ) + self._df["indicator"] = self._df["reporter_line"].apply( + lambda x: ReporterLine(x).parse_indicator() + ) # add session number self.__add_session_number() # add prior exposure - self._df['prior_exposures_to_session_type'] = \ - get_prior_exposures_to_session_type(df=self._df) - self._df['prior_exposures_to_image_set'] = \ - get_prior_exposures_to_image_set(df=self._df) - self._df['prior_exposures_to_omissions'] = \ - get_prior_exposures_to_omissions(df=self._df, - fetch_api=self._fetch_api) + self._df[ + "prior_exposures_to_session_type" + ] = get_prior_exposures_to_session_type(df=self._df) + self._df[ + "prior_exposures_to_image_set" + ] = get_prior_exposures_to_image_set(df=self._df) + self._df[ + "prior_exposures_to_omissions" + ] = get_prior_exposures_to_omissions( + df=self._df, fetch_api=self._fetch_api + ) if self._include_trial_metrics: # add trial metrics @@ -98,48 +107,59 @@ def postprocess_additional(self): target=self._get_trial_metrics_helper, behavior_session_ids=self._df.index.tolist(), lims_engine=db_connection_creator( - fallback_credentials=LIMS_DB_CREDENTIAL_MAP), - progress_bar_title='Getting trial metrics for each session' + fallback_credentials=LIMS_DB_CREDENTIAL_MAP + ), + progress_bar_title="Getting trial metrics for each session", ) trial_metrics = pd.DataFrame(trial_metrics).set_index( - 'behavior_session_id') + "behavior_session_id" + ) self._df = self._df.merge( - trial_metrics, - left_index=True, - right_index=True) + trial_metrics, left_index=True, right_index=True + ) # Add data from ophys session if self._ophys_session_table is not None: # Merge in ophys data - self._df = self._df.reset_index() \ - .merge(self._ophys_session_table.table.reset_index(), - on='behavior_session_id', - how='left', - suffixes=('_behavior', '_ophys')) - self._df = self._df.set_index('behavior_session_id') + self._df = self._df.reset_index().merge( + self._ophys_session_table.table.reset_index(), + on="behavior_session_id", + how="left", + suffixes=("_behavior", "_ophys"), + ) + self._df = self._df.set_index("behavior_session_id") # Prioritize behavior date_of_acquisition - self._df['date_of_acquisition'] = \ - self._df['date_of_acquisition_behavior'] - self._df = self._df.drop(['date_of_acquisition_behavior', - 'date_of_acquisition_ophys'], axis=1) + self._df["date_of_acquisition"] = self._df[ + "date_of_acquisition_behavior" + ] + self._df = self._df.drop( + ["date_of_acquisition_behavior", "date_of_acquisition_ophys"], + axis=1, + ) + # Enforce an integer type on due to there not being a value for + # ophys_session_id for every behavior_session. Pandas defaults to + # NaN here, changing the type to float unless otherwise fixed. + self._df["ophys_session_id"] = self._df["ophys_session_id"].astype( + "Int64" + ) def __add_session_number(self): """Parses session number from session type and and adds to dataframe""" def parse_session_number(session_type: str): """Parse the session number from session type""" - match = re.match(r'OPHYS_(?P\d+)', - session_type) + match = re.match(r"OPHYS_(?P\d+)", session_type) if match is None: return None - return int(match.group('session_number')) + return int(match.group("session_number")) - session_type = self._df['session_type'] + session_type = self._df["session_type"] session_type = session_type[session_type.notnull()] - self._df.loc[session_type.index, 'session_number'] = \ - session_type.apply(parse_session_number) + self._df.loc[ + session_type.index, "session_number" + ] = session_type.apply(parse_session_number) @staticmethod def _get_trial_metrics_helper(*args) -> Dict: @@ -148,12 +168,10 @@ def _get_trial_metrics_helper(*args) -> Dict: behavior_session_id, db_conn = args[0] stimulus_file = BehaviorStimulusFile.from_lims( - behavior_session_id=behavior_session_id, - db=db_conn + behavior_session_id=behavior_session_id, db=db_conn ) stimulus_timestamps = StimulusTimestamps.from_stimulus_file( - stimulus_file=stimulus_file, - monitor_delay=0.0 + stimulus_file=stimulus_file, monitor_delay=0.0 ) trials = Trials.from_stimulus_file( @@ -161,22 +179,22 @@ def _get_trial_metrics_helper(*args) -> Dict: stimulus_timestamps=stimulus_timestamps, licks=Licks.from_stimulus_file( stimulus_file=stimulus_file, - stimulus_timestamps=stimulus_timestamps + stimulus_timestamps=stimulus_timestamps, ), rewards=Rewards.from_stimulus_file( stimulus_file=stimulus_file, - stimulus_timestamps=stimulus_timestamps - ) + stimulus_timestamps=stimulus_timestamps, + ), ) return { - 'behavior_session_id': behavior_session_id, - 'trial_count': trials.trial_count, - 'go_trial_count': trials.go_trial_count, - 'catch_trial_count': trials.catch_trial_count, - 'hit_trial_count': trials.hit_trial_count, - 'miss_trial_count': trials.miss_trial_count, - 'false_alarm_trial_count': trials.false_alarm_trial_count, - 'correct_reject_trial_count': trials.correct_reject_trial_count, - 'engaged_trial_count': trials.get_engaged_trial_count() + "behavior_session_id": behavior_session_id, + "trial_count": trials.trial_count, + "go_trial_count": trials.go_trial_count, + "catch_trial_count": trials.catch_trial_count, + "hit_trial_count": trials.hit_trial_count, + "miss_trial_count": trials.miss_trial_count, + "false_alarm_trial_count": trials.false_alarm_trial_count, + "correct_reject_trial_count": trials.correct_reject_trial_count, + "engaged_trial_count": trials.get_engaged_trial_count(), } diff --git a/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/mouse_id.py b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/mouse_id.py index 8a52bff27..58842673a 100644 --- a/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/mouse_id.py +++ b/allensdk/brain_observatory/behavior/data_objects/metadata/subject_metadata/mouse_id.py @@ -1,26 +1,37 @@ -from pynwb import NWBFile - -from allensdk.core import DataObject -from allensdk.core import \ - JsonReadableInterface, LimsReadableInterface, NwbReadableInterface +from allensdk.core import ( + DataObject, + JsonReadableInterface, + LimsReadableInterface, + NwbReadableInterface, +) from allensdk.internal.api import PostgresQueryMixin +from pynwb import NWBFile -class MouseId(DataObject, LimsReadableInterface, JsonReadableInterface, - NwbReadableInterface): +class MouseId( + DataObject, + LimsReadableInterface, + JsonReadableInterface, + NwbReadableInterface, +): """the LabTracks ID""" - def __init__(self, mouse_id: int): + + def __init__(self, mouse_id: str): super().__init__(name="mouse_id", value=mouse_id) @classmethod def from_json(cls, dict_repr: dict) -> "MouseId": - mouse_id = dict_repr['external_specimen_name'] - mouse_id = int(mouse_id) + mouse_id = dict_repr["external_specimen_name"] + # Check to make sure the dictionary value is string type and if not + # make it so. + if not isinstance(mouse_id, str): + mouse_id = str(mouse_id) return cls(mouse_id=mouse_id) @classmethod - def from_lims(cls, behavior_session_id: int, - lims_db: PostgresQueryMixin) -> "MouseId": + def from_lims( + cls, behavior_session_id: int, lims_db: PostgresQueryMixin + ) -> "MouseId": # TODO: Should this even be included? # Found sometimes there were entries with NONE which is # why they are filtered out; also many entries in the table @@ -33,9 +44,9 @@ def from_lims(cls, behavior_session_id: int, WHERE bs.id={behavior_session_id} AND sp.external_specimen_name IS NOT NULL; """ - mouse_id = int(lims_db.fetchone(query, strict=True)) + mouse_id = lims_db.fetchone(query, strict=True) return cls(mouse_id=mouse_id) @classmethod def from_nwb(cls, nwbfile: NWBFile) -> "MouseId": - return cls(mouse_id=int(nwbfile.subject.subject_id)) + return cls(mouse_id=nwbfile.subject.subject_id) diff --git a/allensdk/test/brain_observatory/behavior/data_objects/metadata/behavior_metadata/test_behavior_metadata.py b/allensdk/test/brain_observatory/behavior/data_objects/metadata/behavior_metadata/test_behavior_metadata.py index da98e1fb6..595429be6 100644 --- a/allensdk/test/brain_observatory/behavior/data_objects/metadata/behavior_metadata/test_behavior_metadata.py +++ b/allensdk/test/brain_observatory/behavior/data_objects/metadata/behavior_metadata/test_behavior_metadata.py @@ -2,58 +2,58 @@ import pickle import uuid from pathlib import Path +from uuid import UUID import pynwb import pytest import pytz -from uuid import UUID - from allensdk.brain_observatory.behavior.data_files import BehaviorStimulusFile from allensdk.brain_observatory.behavior.data_objects import BehaviorSessionId -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .behavior_metadata.behavior_metadata import \ - BehaviorMetadata -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .behavior_metadata.behavior_session_uuid import \ - BehaviorSessionUUID -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .behavior_metadata.date_of_acquisition import \ - DateOfAcquisition -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .behavior_metadata.equipment import \ - Equipment -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .behavior_metadata.session_type import \ - SessionType -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .behavior_metadata.project_code import \ - ProjectCode -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .behavior_metadata.stimulus_frame_rate import \ - StimulusFrameRate -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .subject_metadata.age import \ - Age -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .subject_metadata.driver_line import \ - DriverLine -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .subject_metadata.full_genotype import \ - FullGenotype -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .subject_metadata.mouse_id import \ - MouseId -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .subject_metadata.reporter_line import \ - ReporterLine -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .subject_metadata.sex import \ - Sex -from allensdk.brain_observatory.behavior.data_objects.metadata \ - .subject_metadata.subject_metadata import \ - SubjectMetadata -from allensdk.test.brain_observatory.behavior.data_objects.lims_util import \ - LimsTest +from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_metadata.behavior_metadata import ( # noqa: E501 + BehaviorMetadata, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_metadata.behavior_session_uuid import ( # noqa: E501 + BehaviorSessionUUID, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_metadata.date_of_acquisition import ( # noqa: E501 + DateOfAcquisition, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_metadata.equipment import ( # noqa: E501 + Equipment, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_metadata.project_code import ( # noqa: E501 + ProjectCode, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_metadata.session_type import ( # noqa: E501 + SessionType, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_metadata.stimulus_frame_rate import ( # noqa: E501 + StimulusFrameRate, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.subject_metadata.age import ( # noqa: E501 + Age, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.subject_metadata.driver_line import ( # noqa: E501 + DriverLine, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.subject_metadata.full_genotype import ( # noqa: E501 + FullGenotype, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.subject_metadata.mouse_id import ( # noqa: E501 + MouseId, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.subject_metadata.reporter_line import ( # noqa: E501 + ReporterLine, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.subject_metadata.sex import ( # noqa: E501 + Sex, +) +from allensdk.brain_observatory.behavior.data_objects.metadata.subject_metadata.subject_metadata import ( # noqa: E501 + SubjectMetadata, +) +from allensdk.test.brain_observatory.behavior.data_objects.lims_util import ( + LimsTest, +) class BehaviorMetaTestCase: @@ -64,28 +64,31 @@ def setup_class(cls): @staticmethod def _get_meta(): subject_meta = SubjectMetadata( - sex=Sex(sex='M'), + sex=Sex(sex="M"), age=Age(age=139), reporter_line=ReporterLine(reporter_line="Ai93(TITL-GCaMP6f)"), full_genotype=FullGenotype( full_genotype="Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;" - "Ai93(TITL-GCaMP6f)/wt"), + "Ai93(TITL-GCaMP6f)/wt" + ), driver_line=DriverLine( - driver_line=["Camk2a-tTA", "Slc17a7-IRES2-Cre"]), - mouse_id=MouseId(mouse_id=416369) - + driver_line=["Camk2a-tTA", "Slc17a7-IRES2-Cre"] + ), + mouse_id=MouseId(mouse_id="416369"), ) behavior_meta = BehaviorMetadata( subject_metadata=subject_meta, behavior_session_id=BehaviorSessionId(behavior_session_id=4242), - equipment=Equipment(equipment_name='my_device'), + equipment=Equipment(equipment_name="my_device"), stimulus_frame_rate=StimulusFrameRate(stimulus_frame_rate=60.0), - session_type=SessionType(session_type='Unknown'), + session_type=SessionType(session_type="Unknown"), behavior_session_uuid=BehaviorSessionUUID( - behavior_session_uuid=uuid.uuid4()), + behavior_session_uuid=uuid.uuid4() + ), date_of_acquisition=DateOfAcquisition( - datetime.datetime(2022, 8, 24, 12, 35)), - project_code=ProjectCode('1234') + datetime.datetime(2022, 8, 24, 12, 35) + ), + project_code=ProjectCode("1234"), ) return behavior_meta @@ -96,29 +99,34 @@ def test_behavior_session_uuid(self): behavior_session_id = 823847007 meta = BehaviorMetadata.from_lims( behavior_session_id=BehaviorSessionId( - behavior_session_id=behavior_session_id), - lims_db=self.dbconn + behavior_session_id=behavior_session_id + ), + lims_db=self.dbconn, + ) + assert meta.behavior_session_uuid == uuid.UUID( + "394a910e-94c7-4472-9838-5345aff59ed8" ) - assert meta.behavior_session_uuid == \ - uuid.UUID('394a910e-94c7-4472-9838-5345aff59ed8') class TestBehaviorMetadata(BehaviorMetaTestCase): def test_cre_line(self): """Tests that cre_line properly parsed from driver_line""" fg = FullGenotype( - full_genotype='Sst-IRES-Cre/wt;Ai148(TIT2L-GC6f-ICL-tTA2)/wt') - assert fg.parse_cre_line() == 'Sst-IRES-Cre' + full_genotype="Sst-IRES-Cre/wt;Ai148(TIT2L-GC6f-ICL-tTA2)/wt" + ) + assert fg.parse_cre_line() == "Sst-IRES-Cre" def test_cre_line_bad_full_genotype(self): """Test that cre_line is None and no error raised""" - fg = FullGenotype(full_genotype='foo') + fg = FullGenotype(full_genotype="foo") with pytest.warns(UserWarning) as record: cre_line = fg.parse_cre_line(warn=True) assert cre_line is None - assert str(record[0].message) == 'Unable to parse cre_line from ' \ - 'full_genotype' + assert ( + str(record[0].message) == "Unable to parse cre_line from " + "full_genotype" + ) def test_cre_line_full_genotype_is_none(self): """Test that cre_line is None and no error raised""" @@ -127,50 +135,69 @@ def test_cre_line_full_genotype_is_none(self): with pytest.warns(UserWarning) as record: cre_line = fg.parse_cre_line(warn=True) assert cre_line is None - assert str(record[0].message) == 'Unable to parse cre_line from ' \ - 'full_genotype' + assert ( + str(record[0].message) == "Unable to parse cre_line from " + "full_genotype" + ) def test_reporter_line(self): """Test that reporter line properly parsed from list""" - reporter_line = ReporterLine.parse(reporter_line=['foo']) - assert reporter_line == 'foo' + reporter_line = ReporterLine.parse(reporter_line=["foo"]) + assert reporter_line == "foo" def test_reporter_line_str(self): """Test that reporter line returns itself if str""" - reporter_line = ReporterLine.parse(reporter_line='foo') - assert reporter_line == 'foo' - - @pytest.mark.parametrize("input_reporter_line, warning_msg, expected", ( - (('foo', 'bar'), 'More than 1 reporter line. ' - 'Returning the first one', 'foo'), - (None, 'Error parsing reporter line. It is null.', None), - ([], 'Error parsing reporter line. The array is empty', None) + reporter_line = ReporterLine.parse(reporter_line="foo") + assert reporter_line == "foo" + + @pytest.mark.parametrize( + "input_reporter_line, warning_msg, expected", + ( + ( + ("foo", "bar"), + "More than 1 reporter line. " "Returning the first one", + "foo", + ), + (None, "Error parsing reporter line. It is null.", None), + ([], "Error parsing reporter line. The array is empty", None), + ), ) - ) - def test_reporter_edge_cases(self, input_reporter_line, warning_msg, - expected): + def test_reporter_edge_cases( + self, input_reporter_line, warning_msg, expected + ): """Test reporter line edge cases""" with pytest.warns(UserWarning) as record: reporter_line = ReporterLine.parse( - reporter_line=input_reporter_line, - warn=True) + reporter_line=input_reporter_line, warn=True + ) assert reporter_line == expected assert str(record[0].message) == warning_msg def test_age_in_days(self): """Test that age_in_days properly parsed from age""" - age = Age._age_code_to_days(age='P123') + age = Age._age_code_to_days(age="P123") assert age == 123 - @pytest.mark.parametrize("input_age, warning_msg, expected", ( - ('unkown', 'Could not parse numeric age from age code ' - '(age code does not start with "P")', None), - ('P', 'Could not parse numeric age from age code ' - '(no numeric values found in age code)', None) + @pytest.mark.parametrize( + "input_age, warning_msg, expected", + ( + ( + "unkown", + "Could not parse numeric age from age code " + '(age code does not start with "P")', + None, + ), + ( + "P", + "Could not parse numeric age from age code " + "(no numeric values found in age code)", + None, + ), + ), ) - ) - def test_age_in_days_edge_cases(self, monkeypatch, input_age, warning_msg, - expected): + def test_age_in_days_edge_cases( + self, monkeypatch, input_age, warning_msg, expected + ): """Test age in days edge cases""" with pytest.warns(UserWarning) as record: age_in_days = Age._age_code_to_days(age=input_age, warn=True) @@ -178,88 +205,112 @@ def test_age_in_days_edge_cases(self, monkeypatch, input_age, warning_msg, assert age_in_days is None assert str(record[0].message) == warning_msg - @pytest.mark.parametrize("test_params, expected_warn_msg", [ - # Vanilla test case - ({ - "extractor_expt_date": datetime.datetime.strptime( - "2021-03-14 03:14:15", - "%Y-%m-%d %H:%M:%S"), - "pkl_expt_date": datetime.datetime.strptime("2021-03-14 03:14:15", - "%Y-%m-%d %H:%M:%S"), - "behavior_session_id": 1 - }, None), - - # pkl expt date stored in unix format - ({ - "extractor_expt_date": datetime.datetime.strptime( - "2021-03-14 03:14:15", - "%Y-%m-%d %H:%M:%S"), - "pkl_expt_date": 1615716855.0, - "behavior_session_id": 2 - }, None), - - # Extractor and pkl dates differ significantly - ({ - "extractor_expt_date": datetime.datetime.strptime( - "2021-03-14 03:14:15", - "%Y-%m-%d %H:%M:%S"), - "pkl_expt_date": datetime.datetime.strptime("2021-03-14 20:14:15", - "%Y-%m-%d %H:%M:%S"), - "behavior_session_id": 3 - }, - "The `date_of_acquisition` field in LIMS *"), - - # pkl file contains an unparseable datetime - ({ - "extractor_expt_date": datetime.datetime.strptime( - "2021-03-14 03:14:15", - "%Y-%m-%d %H:%M:%S"), - "pkl_expt_date": None, - "behavior_session_id": 4 - }, - "Could not parse the acquisition datetime *"), - ]) - def test_get_date_of_acquisition(self, tmp_path, test_params, - expected_warn_msg): + @pytest.mark.parametrize( + "test_params, expected_warn_msg", + [ + # Vanilla test case + ( + { + "extractor_expt_date": datetime.datetime.strptime( + "2021-03-14 03:14:15", "%Y-%m-%d %H:%M:%S" + ), + "pkl_expt_date": datetime.datetime.strptime( + "2021-03-14 03:14:15", "%Y-%m-%d %H:%M:%S" + ), + "behavior_session_id": 1, + }, + None, + ), + # pkl expt date stored in unix format + ( + { + "extractor_expt_date": datetime.datetime.strptime( + "2021-03-14 03:14:15", "%Y-%m-%d %H:%M:%S" + ), + "pkl_expt_date": 1615716855.0, + "behavior_session_id": 2, + }, + None, + ), + # Extractor and pkl dates differ significantly + ( + { + "extractor_expt_date": datetime.datetime.strptime( + "2021-03-14 03:14:15", "%Y-%m-%d %H:%M:%S" + ), + "pkl_expt_date": datetime.datetime.strptime( + "2021-03-14 20:14:15", "%Y-%m-%d %H:%M:%S" + ), + "behavior_session_id": 3, + }, + "The `date_of_acquisition` field in LIMS *", + ), + # pkl file contains an unparseable datetime + ( + { + "extractor_expt_date": datetime.datetime.strptime( + "2021-03-14 03:14:15", "%Y-%m-%d %H:%M:%S" + ), + "pkl_expt_date": None, + "behavior_session_id": 4, + }, + "Could not parse the acquisition datetime *", + ), + ], + ) + def test_get_date_of_acquisition( + self, tmp_path, test_params, expected_warn_msg + ): mock_session_id = test_params["behavior_session_id"] pkl_save_path = tmp_path / f"mock_pkl_{mock_session_id}.pkl" - with open(pkl_save_path, 'wb') as handle: - pickle.dump({"start_time": test_params['pkl_expt_date']}, handle) + with open(pkl_save_path, "wb") as handle: + pickle.dump({"start_time": test_params["pkl_expt_date"]}, handle) tz = pytz.timezone("America/Los_Angeles") extractor_expt_date = tz.localize( - test_params['extractor_expt_date']).astimezone(pytz.utc) + test_params["extractor_expt_date"] + ).astimezone(pytz.utc) stimulus_file = BehaviorStimulusFile(filepath=pkl_save_path) - obt_date = DateOfAcquisition( - date_of_acquisition=extractor_expt_date) + obt_date = DateOfAcquisition(date_of_acquisition=extractor_expt_date) if expected_warn_msg: with pytest.warns(Warning, match=expected_warn_msg): obt_date.validate( stimulus_file=stimulus_file, - behavior_session_id=test_params['behavior_session_id']) + behavior_session_id=test_params["behavior_session_id"], + ) assert obt_date.value == extractor_expt_date def test_indicator(self): """Test that indicator is parsed from full_genotype""" reporter_line = ReporterLine( - reporter_line='Ai148(TIT2L-GC6f-ICL-tTA2)') - assert reporter_line.parse_indicator() == 'GCaMP6f' - - @pytest.mark.parametrize("input_reporter_line, warning_msg, expected", ( - (None, - 'Could not parse indicator from reporter because there is no ' - 'reporter', None), - ('foo', 'Could not parse indicator from reporter because none' - 'of the expected substrings were found in the reporter', - None) + reporter_line="Ai148(TIT2L-GC6f-ICL-tTA2)" + ) + assert reporter_line.parse_indicator() == "GCaMP6f" + + @pytest.mark.parametrize( + "input_reporter_line, warning_msg, expected", + ( + ( + None, + "Could not parse indicator from reporter because there is no " + "reporter", + None, + ), + ( + "foo", + "Could not parse indicator from reporter because none" + "of the expected substrings were found in the reporter", + None, + ), + ), ) - ) - def test_indicator_edge_cases(self, input_reporter_line, warning_msg, - expected): + def test_indicator_edge_cases( + self, input_reporter_line, warning_msg, expected + ): """Test indicator parsing edge cases""" with pytest.warns(UserWarning) as record: reporter_line = ReporterLine(reporter_line=input_reporter_line) @@ -270,22 +321,26 @@ def test_indicator_edge_cases(self, input_reporter_line, warning_msg, class TestBehaviorStimulusFile: """Tests properties read from stimulus file""" + def setup_class(cls): dir = Path(__file__).parent.parent.parent.resolve() - test_data_dir = dir / 'test_data' - sf_path = test_data_dir / 'stimulus_file.pkl' + test_data_dir = dir / "test_data" + sf_path = test_data_dir / "stimulus_file.pkl" cls.stimulus_file = BehaviorStimulusFile.from_json( - dict_repr={'behavior_stimulus_file': str(sf_path)}) + dict_repr={"behavior_stimulus_file": str(sf_path)} + ) def test_session_uuid(self): uuid = BehaviorSessionUUID.from_stimulus_file( - stimulus_file=self.stimulus_file) - expected = UUID('138531ab-fe59-4523-9154-07c8d97bbe03') + stimulus_file=self.stimulus_file + ) + expected = UUID("138531ab-fe59-4523-9154-07c8d97bbe03") assert expected == uuid.value def test_get_stimulus_frame_rate(self): rate = StimulusFrameRate.from_stimulus_file( - stimulus_file=self.stimulus_file) + stimulus_file=self.stimulus_file + ) assert 62.0 == rate.value @@ -293,25 +348,28 @@ def test_date_of_acquisition_utc(): """Tests that when read from json (in Pacific time), that date of acquisition is converted to utc""" expected = DateOfAcquisition( - date_of_acquisition=datetime.datetime(2019, 9, 26, 16, - tzinfo=pytz.UTC)) + date_of_acquisition=datetime.datetime(2019, 9, 26, 16, tzinfo=pytz.UTC) + ) actual = DateOfAcquisition.from_json( - dict_repr={'date_of_acquisition': '2019-09-26 09:00:00'}) + dict_repr={"date_of_acquisition": "2019-09-26 09:00:00"} + ) assert expected == actual class TestNWB(BehaviorMetaTestCase): def setup_method(self, method): self.nwbfile = pynwb.NWBFile( - session_description='asession', - identifier='afile', - session_start_time=datetime.datetime(2022, 8, 24, 12, 35, - tzinfo=pytz.UTC) + session_description="asession", + identifier="afile", + session_start_time=datetime.datetime( + 2022, 8, 24, 12, 35, tzinfo=pytz.UTC + ), ) - @pytest.mark.parametrize('roundtrip', [True, False]) - def test_add_behavior_only_metadata(self, roundtrip, - data_object_roundtrip_fixture): + @pytest.mark.parametrize("roundtrip", [True, False]) + def test_add_behavior_only_metadata( + self, roundtrip, data_object_roundtrip_fixture + ): self.meta.to_nwb(nwbfile=self.nwbfile) if roundtrip: @@ -320,5 +378,4 @@ def test_add_behavior_only_metadata(self, roundtrip, ) else: meta_obt = BehaviorMetadata.from_nwb(nwbfile=self.nwbfile) - assert self.meta == meta_obt diff --git a/allensdk/test/brain_observatory/behavior/data_objects/metadata/test_behavior_ophys_metadata.py b/allensdk/test/brain_observatory/behavior/data_objects/metadata/test_behavior_ophys_metadata.py index 261791e7c..c93b6f010 100644 --- a/allensdk/test/brain_observatory/behavior/data_objects/metadata/test_behavior_ophys_metadata.py +++ b/allensdk/test/brain_observatory/behavior/data_objects/metadata/test_behavior_ophys_metadata.py @@ -5,21 +5,44 @@ import pynwb import pytest import pytz - -from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_metadata.equipment import Equipment # NOQA -from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_ophys_metadata import BehaviorOphysMetadata # NOQA -from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.field_of_view_shape import FieldOfViewShape # NOQA -from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.imaging_depth import ImagingDepth # NOQA -from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.multi_plane_metadata.imaging_plane_group import ImagingPlaneGroup # NOQA -from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.multi_plane_metadata.multi_plane_metadata import MultiplaneMetadata # NOQA -from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.ophys_container_id import OphysContainerId # NOQA -from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.ophys_experiment_metadata import OphysExperimentMetadata # NOQA -from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.ophys_session_id import OphysSessionId # NOQA -from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.ophys_project_code import OphysProjectCode # NOQA -from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.targeted_imaging_depth import TargetedImagingDepth # NOQA +from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_metadata.equipment import ( # noqa: E501 + Equipment, +) # NOQA +from allensdk.brain_observatory.behavior.data_objects.metadata.behavior_ophys_metadata import ( # noqa: E501 + BehaviorOphysMetadata, +) # NOQA +from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.field_of_view_shape import ( # noqa: E501 + FieldOfViewShape, +) # NOQA +from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.imaging_depth import ( # noqa: E501 + ImagingDepth, +) # NOQA +from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.multi_plane_metadata.imaging_plane_group import ( # noqa: E501 + ImagingPlaneGroup, +) # NOQA +from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.multi_plane_metadata.multi_plane_metadata import ( # noqa: E501 + MultiplaneMetadata, +) # NOQA +from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.ophys_container_id import ( # noqa: E501 + OphysContainerId, +) # NOQA +from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.ophys_experiment_metadata import ( # noqa: E501 + OphysExperimentMetadata, +) # NOQA +from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.ophys_project_code import ( # noqa: E501 + OphysProjectCode, +) # NOQA +from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.ophys_session_id import ( # noqa: E501 + OphysSessionId, +) # NOQA +from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.targeted_imaging_depth import ( # noqa: E501 + TargetedImagingDepth, +) # NOQA from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP from allensdk.internal.api import db_connection_creator -from allensdk.test.brain_observatory.behavior.data_objects.metadata.behavior_metadata.test_behavior_metadata import TestBehaviorMetadata # NOQA +from allensdk.test.brain_observatory.behavior.data_objects.metadata.behavior_metadata.test_behavior_metadata import ( # noqa: E501 + TestBehaviorMetadata, +) # NOQA class TestBOM: @@ -35,122 +58,137 @@ def _get_meta(): ophys_meta = OphysExperimentMetadata( ophys_experiment_id=1234, ophys_session_id=OphysSessionId(session_id=999), - ophys_container_id=OphysContainerId( - ophys_container_id=5678), + ophys_container_id=OphysContainerId(ophys_container_id=5678), field_of_view_shape=FieldOfViewShape(width=4, height=4), imaging_depth=ImagingDepth(imaging_depth=375), targeted_imaging_depth=TargetedImagingDepth( - targeted_imaging_depth=375), - project_code=OphysProjectCode('1234') + targeted_imaging_depth=375 + ), + project_code=OphysProjectCode("1234"), ) behavior_metadata = TestBehaviorMetadata() behavior_metadata.setup_class() return BehaviorOphysMetadata( - behavior_metadata=behavior_metadata.meta, - ophys_metadata=ophys_meta + behavior_metadata=behavior_metadata.meta, ophys_metadata=ophys_meta ) def _get_multiplane_meta(self): bo_meta = self.meta - bo_meta.behavior_metadata._equipment = \ - Equipment(equipment_name='MESO.1') + bo_meta.behavior_metadata._equipment = Equipment( + equipment_name="MESO.1" + ) ophys_experiment_metadata = bo_meta.ophys_metadata - imaging_plane_group = ImagingPlaneGroup(plane_group_count=5, - plane_group=0) + imaging_plane_group = ImagingPlaneGroup( + plane_group_count=5, plane_group=0 + ) multiplane_meta = MultiplaneMetadata( ophys_experiment_id=ophys_experiment_metadata.ophys_experiment_id, ophys_session_id=ophys_experiment_metadata._ophys_session_id, ophys_container_id=ophys_experiment_metadata._ophys_container_id, field_of_view_shape=ophys_experiment_metadata._field_of_view_shape, imaging_depth=ophys_experiment_metadata._imaging_depth, - targeted_imaging_depth=ophys_experiment_metadata. - _targeted_imaging_depth, + targeted_imaging_depth=ophys_experiment_metadata._targeted_imaging_depth, # noqa: E501 project_code=ophys_experiment_metadata._project_code, - imaging_plane_group=imaging_plane_group + imaging_plane_group=imaging_plane_group, ) return BehaviorOphysMetadata( behavior_metadata=bo_meta.behavior_metadata, - ophys_metadata=multiplane_meta + ophys_metadata=multiplane_meta, ) class TestInternal(TestBOM): @classmethod def setup_method(self, method): - marks = getattr(method, 'pytestmark', None) + marks = getattr(method, "pytestmark", None) if marks: marks = [m.name for m in marks] # Will only create a dbconn if the test requires_bamboo - if 'requires_bamboo' in marks: + if "requires_bamboo" in marks: self.dbconn = db_connection_creator( - fallback_credentials=LIMS_DB_CREDENTIAL_MAP) + fallback_credentials=LIMS_DB_CREDENTIAL_MAP + ) @pytest.mark.requires_bamboo - @pytest.mark.parametrize('meso', [True, False]) + @pytest.mark.parametrize("meso", [True, False]) def test_from_lims(self, meso): if meso: ophys_experiment_id = 951980471 else: ophys_experiment_id = 994278291 bom = BehaviorOphysMetadata.from_lims( - ophys_experiment_id=ophys_experiment_id, lims_db=self.dbconn, - is_multiplane=meso) + ophys_experiment_id=ophys_experiment_id, + lims_db=self.dbconn, + is_multiplane=meso, + ) if meso: - assert isinstance(bom.ophys_metadata, - MultiplaneMetadata) + assert isinstance(bom.ophys_metadata, MultiplaneMetadata) assert bom.ophys_metadata.imaging_depth == 150 assert bom.ophys_metadata.targeted_imaging_depth == 150 - assert bom.behavior_metadata.session_type == 'OPHYS_1_images_A' - assert bom.behavior_metadata.subject_metadata.reporter_line == \ - 'Ai148(TIT2L-GC6f-ICL-tTA2)' - assert bom.behavior_metadata.subject_metadata.driver_line == \ - ['Sst-IRES-Cre'] - assert bom.behavior_metadata.subject_metadata.mouse_id == 457841 - assert bom.behavior_metadata.subject_metadata.full_genotype == \ - 'Sst-IRES-Cre/wt;Ai148(TIT2L-GC6f-ICL-tTA2)/wt' + assert bom.behavior_metadata.session_type == "OPHYS_1_images_A" + assert ( + bom.behavior_metadata.subject_metadata.reporter_line + == "Ai148(TIT2L-GC6f-ICL-tTA2)" + ) + assert bom.behavior_metadata.subject_metadata.driver_line == [ + "Sst-IRES-Cre" + ] + assert bom.behavior_metadata.subject_metadata.mouse_id == "457841" + assert ( + bom.behavior_metadata.subject_metadata.full_genotype + == "Sst-IRES-Cre/wt;Ai148(TIT2L-GC6f-ICL-tTA2)/wt" + ) assert bom.behavior_metadata.subject_metadata.age_in_days == 233 - assert bom.behavior_metadata.subject_metadata.sex == 'F' + assert bom.behavior_metadata.subject_metadata.sex == "F" else: assert isinstance(bom.ophys_metadata, OphysExperimentMetadata) assert bom.ophys_metadata.imaging_depth == 175 assert bom.ophys_metadata.targeted_imaging_depth == 175 - assert bom.behavior_metadata.session_type == 'OPHYS_4_images_A' - assert bom.behavior_metadata.subject_metadata.reporter_line == \ - 'Ai93(TITL-GCaMP6f)' - assert bom.behavior_metadata.subject_metadata.driver_line == \ - ['Camk2a-tTA', 'Slc17a7-IRES2-Cre'] - assert bom.behavior_metadata.subject_metadata.mouse_id == 491060 - assert bom.behavior_metadata.subject_metadata.full_genotype == \ - 'Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai93(TITL-GCaMP6f)/wt' + assert bom.behavior_metadata.session_type == "OPHYS_4_images_A" + assert ( + bom.behavior_metadata.subject_metadata.reporter_line + == "Ai93(TITL-GCaMP6f)" + ) + assert bom.behavior_metadata.subject_metadata.driver_line == [ + "Camk2a-tTA", + "Slc17a7-IRES2-Cre", + ] + assert bom.behavior_metadata.subject_metadata.mouse_id == "491060" + assert ( + bom.behavior_metadata.subject_metadata.full_genotype + == "Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai93(TITL-GCaMP6f)/wt" + ) assert bom.behavior_metadata.subject_metadata.age_in_days == 130 - assert bom.behavior_metadata.subject_metadata.sex == 'M' + assert bom.behavior_metadata.subject_metadata.sex == "M" class TestJson(TestBOM): @classmethod def setup_method(self, method): dir = Path(__file__).parent.resolve() - test_data_dir = dir.parent / 'test_data' - with open(test_data_dir / 'test_input.json') as f: + test_data_dir = dir.parent / "test_data" + with open(test_data_dir / "test_input.json") as f: dict_repr = json.load(f) - dict_repr = dict_repr['session_data'] - dict_repr['sync_file'] = str(test_data_dir / 'sync.h5') - dict_repr['behavior_stimulus_file'] = str(test_data_dir / - 'behavior_stimulus_file.pkl') - dict_repr['dff_file'] = str(test_data_dir / 'demix_file.h5') + dict_repr = dict_repr["session_data"] + dict_repr["sync_file"] = str(test_data_dir / "sync.h5") + dict_repr["behavior_stimulus_file"] = str( + test_data_dir / "behavior_stimulus_file.pkl" + ) + dict_repr["dff_file"] = str(test_data_dir / "demix_file.h5") self.dict_repr = dict_repr @pytest.mark.requires_bamboo - @pytest.mark.parametrize('meso', [True, False]) + @pytest.mark.parametrize("meso", [True, False]) def test_from_json(self, meso): if meso: - self.dict_repr['rig_name'] = 'MESO.1' - bom = BehaviorOphysMetadata.from_json(dict_repr=self.dict_repr, - is_multiplane=meso) + self.dict_repr["rig_name"] = "MESO.1" + bom = BehaviorOphysMetadata.from_json( + dict_repr=self.dict_repr, is_multiplane=meso + ) if meso: assert isinstance(bom.ophys_metadata, MultiplaneMetadata) @@ -162,16 +200,18 @@ class TestNWB(TestBOM): def setup_method(self, method): self.meta = self._get_meta() self.nwbfile = pynwb.NWBFile( - session_description='asession', + session_description="asession", identifier=str(self.meta.ophys_metadata.ophys_experiment_id), - session_start_time=datetime.datetime(2022, 8, 24, 12, 35, - tzinfo=pytz.UTC) + session_start_time=datetime.datetime( + 2022, 8, 24, 12, 35, tzinfo=pytz.UTC + ), ) - @pytest.mark.parametrize('meso', [True, False]) - @pytest.mark.parametrize('roundtrip', [True, False]) - def test_read_write_nwb(self, roundtrip, - data_object_roundtrip_fixture, meso): + @pytest.mark.parametrize("meso", [True, False]) + @pytest.mark.parametrize("roundtrip", [True, False]) + def test_read_write_nwb( + self, roundtrip, data_object_roundtrip_fixture, meso + ): if meso: self.meta = self._get_multiplane_meta() @@ -181,9 +221,9 @@ def test_read_write_nwb(self, roundtrip, obt = data_object_roundtrip_fixture( nwbfile=self.nwbfile, data_object_cls=BehaviorOphysMetadata, - is_multiplane=meso) + is_multiplane=meso, + ) else: - obt = self.meta.from_nwb(nwbfile=self.nwbfile, - is_multiplane=meso) + obt = self.meta.from_nwb(nwbfile=self.nwbfile, is_multiplane=meso) assert obt == self.meta