Merge branch 'ticket/2541/dev' into rc/2.15.0

allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_lims_api.py

@@ -21,6 +21,7 @@
     build_in_list_selector_query, build_where_clause)
 from allensdk.internal.brain_observatory.util.multi_session_utils import \
     get_session_metadata_multiprocessing
+from allensdk.brain_observatory.behavior.data_objects import BehaviorSessionId
 
 
 class BehaviorProjectLimsApi(BehaviorProjectBase):
@@ -236,6 +237,7 @@ def _get_behavior_summary_table(self) -> pd.DataFrame:
         query = f"""
             SELECT
                 bs.id AS behavior_session_id,
+                pr.code as project_code,
                 equipment.name as equipment_name,
                 bs.date_of_acquisition,
                 d.id as donor_id,
@@ -257,6 +259,7 @@ def _get_behavior_summary_table(self) -> pd.DataFrame:
                 {self._build_line_from_donor_query("driver")}
             ) driver on driver.donor_id = d.id
             LEFT OUTER JOIN equipment ON equipment.id = bs.equipment_id
+            LEFT OUTER JOIN projects pr ON pr.id = bs.project_id
         """
 
         if self.data_release_date is not None:
@@ -416,7 +419,7 @@ def _get_ophys_session_table(self) -> pd.DataFrame:
         """Helper function for easier testing.
         Return a pd.Dataframe table with all ophys_session_ids and relevant
         metadata.
-        Return columns: ophys_session_id, behavior_session_id,
+        Return columns: ophys_session_id, behavior_session_id, project_code,
                         ophys_experiment_id, project_code, session_name,
                         date_of_acquisition,
                         specimen_id, full_genotype, sex, age_in_days,
@@ -428,10 +431,10 @@ def _get_ophys_session_table(self) -> pd.DataFrame:
             SELECT
                 os.id as ophys_session_id,
                 bs.id as behavior_session_id,
+                pr.code as project_code,
                 imaging_plane_group_count.imaging_plane_group_count,
                 exp_ids.experiment_ids as ophys_experiment_id,
                 cntr_ids.container_ids as ophys_container_id,
-                pr.code as project_code,
                 os.name as session_name,
                 os.date_of_acquisition,
                 os.specimen_id
@@ -528,7 +531,7 @@ def get_behavior_session_table(self, n_workers: int = 1) -> pd.DataFrame:
         else:
             session_metadata = [
                 BehaviorMetadata.from_lims(
-                    behavior_session_id=behavior_session_id,
+                    behavior_session_id=BehaviorSessionId(behavior_session_id),
                     lims_db=db_connection_creator(
                         fallback_credentials=LIMS_DB_CREDENTIAL_MAP
                     )

allensdk/brain_observatory/behavior/behavior_project_cache/tables/ophys_mixin.py

@@ -1,3 +1,6 @@
+import warnings
+
+
 class OphysMixin:
     """A mixin class for ophys project data"""
     def __init__(self):
@@ -11,3 +14,46 @@ def __init__(self):
             self._df = self._df.drop(
                 ['date_of_acquisition_behavior',
                  'date_of_acquisition_ophys'], axis=1)
+        self._clean_up_project_code()
+
+    def _clean_up_project_code(self):
+        """Remove duplicate project_code columns from the data frames. This is
+        as depending on the table we get the project_code either through the
+        behavior_sessions or ophys_sessions tables.
+
+        Additionally test that the values in the columns are identical.
+        """
+
+        if 'project_code_behavior' in self._df and \
+                'project_code_ophys' in self._df:
+
+            if (self._df['project_code_ophys'].isna()).sum() == 0:
+                # If there are no missing ophys_session_ids for the table then
+                # we are loading of the ophys tables and should be able to
+                # compare the ids directly to assure ourselves that the
+                # project ids are the same between ophys and behavior sessions.
+                if not self._df['project_code_ophys'].equals(
+                        self._df['project_code_behavior']):
+                    warnings.warn("BehaviorSession and OphysSession "
+                                  "project_code's do not agree. This is "
+                                  "likely due to issues with the data in "
+                                  "LIMS. Using OphysSession project_code.")
+                self._df['project_code'] = self._df['project_code_ophys']
+            else:
+                # If there are missing ophys_session_ids for the table then
+                # we are loading of the behavior table first will need to mask
+                # to only the sessions that have ophys_session_ids before
+                # comparing project_codes.
+                has_ophys_session_mask = ~self._df['project_code_ophys'].isna()
+                if not self._df['project_code_behavior'][
+                        has_ophys_session_mask].equals(
+                            self._df['project_code_ophys'][
+                                has_ophys_session_mask]):
+                    warnings.warn("BehaviorSession and OphysSession "
+                                  "project_codes do not agree. This is likely "
+                                  "due to issues with the data in LIMS. Using "
+                                  "BehaviorSession project_code.")
+                self._df['project_code'] = self._df['project_code_behavior']
+            self._df = self._df.drop(
+                ['project_code_ophys',  'project_code_behavior'],
+                axis=1)

allensdk/brain_observatory/behavior/behavior_project_cache/tables/sessions_table.py

@@ -33,9 +33,12 @@
 from allensdk.internal.api import db_connection_creator
 from allensdk.internal.brain_observatory.util.multi_session_utils import \
     multiprocessing_helper
+from allensdk.brain_observatory.behavior.behavior_project_cache.tables\
+    .ophys_mixin import \
+    OphysMixin
 
 
-class SessionsTable(ProjectTable):
+class SessionsTable(ProjectTable, OphysMixin):
     """Class for storing and manipulating project-level data
     at the session level"""
 
@@ -65,7 +68,8 @@ def __init__(
         self._fetch_api = fetch_api
         self._ophys_session_table = ophys_session_table
         self._include_trial_metrics = include_trial_metrics
-        super().__init__(df=df, suppress=suppress)
+        ProjectTable.__init__(self, df=df, suppress=suppress)
+        OphysMixin.__init__(self)
 
     def postprocess_additional(self):
         # Add subject metadata

allensdk/brain_observatory/behavior/behavior_session.py

@@ -1225,6 +1225,8 @@ def metadata(self) -> Dict[str, Any]:
                     full genotype of transgenic mouse
                 mouse_id: (int)
                     unique identifier for a mouse
+                project_code: (string)
+                    String of project session is associated with.
                 reporter_line: (string)
                     reporter line for a transgenic mouse
                 session_type: (string)
@@ -1469,6 +1471,7 @@ def _get_metadata(self, behavior_metadata: BehaviorMetadata) -> dict:
             'behavior_session_uuid': behavior_metadata.behavior_session_uuid,
             'driver_line': behavior_metadata.subject_metadata.driver_line,
             'mouse_id': behavior_metadata.subject_metadata.mouse_id,
+            'project_code': behavior_metadata.project_code,
             'full_genotype': behavior_metadata.subject_metadata.full_genotype,
             'behavior_session_id': behavior_metadata.behavior_session_id
         }

allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/behavior_metadata.py

@@ -26,6 +26,9 @@
 from allensdk.brain_observatory.behavior.data_objects.metadata\
     .behavior_metadata.foraging_id import \
     ForagingId
+from allensdk.brain_observatory.behavior.data_objects.metadata\
+    .behavior_metadata.project_code import \
+    ProjectCode
 from allensdk.brain_observatory.behavior.data_objects.metadata\
     .behavior_metadata.session_type import \
     SessionType
@@ -191,6 +194,7 @@ def __init__(self,
                  stimulus_frame_rate: StimulusFrameRate,
                  session_type: SessionType,
                  behavior_session_uuid: BehaviorSessionUUID,
+                 project_code: ProjectCode = ProjectCode(),
                  session_duration: Optional[float] = None
                  ):
         super().__init__(name='behavior_metadata', value=None,
@@ -202,6 +206,7 @@ def __init__(self,
         self._stimulus_frame_rate = stimulus_frame_rate
         self._session_type = session_type
         self._behavior_session_uuid = behavior_session_uuid
+        self._project_code = project_code
         self._session_duration = session_duration
 
         self._exclude_from_equals = set()
@@ -238,6 +243,10 @@ def from_lims(
                       foraging_id=foraging_id.value,
                       stimulus_file=stimulus_file)
 
+        project_code = ProjectCode.from_lims(
+            behavior_session_id=behavior_session_id.value,
+            lims_db=lims_db)
+
         return BehaviorMetadata(
             date_of_acquisition=date_of_acquisition,
             subject_metadata=subject_metadata,
@@ -246,6 +255,7 @@ def from_lims(
             stimulus_frame_rate=stimulus_frame_rate,
             session_type=session_type,
             behavior_session_uuid=behavior_session_uuid,
+            project_code=project_code,
             session_duration=stimulus_file.session_duration
         )
 
@@ -273,7 +283,8 @@ def from_json(cls, dict_repr: dict) -> "BehaviorMetadata":
             stimulus_frame_rate=stimulus_frame_rate,
             session_type=session_type,
             behavior_session_uuid=session_uuid,
-            session_duration=stimulus_file.session_duration
+            session_duration=stimulus_file.session_duration,
+            project_code=ProjectCode(),
         )
 
     @classmethod
@@ -286,6 +297,7 @@ def from_nwb(cls, nwbfile: NWBFile) -> "BehaviorMetadata":
         stimulus_frame_rate = StimulusFrameRate.from_nwb(nwbfile=nwbfile)
         session_type = SessionType.from_nwb(nwbfile=nwbfile)
         session_uuid = BehaviorSessionUUID.from_nwb(nwbfile=nwbfile)
+        project_code = ProjectCode.from_nwb(nwbfile=nwbfile)
 
         return BehaviorMetadata(
             date_of_acquisition=date_of_acquisition,
@@ -294,7 +306,8 @@ def from_nwb(cls, nwbfile: NWBFile) -> "BehaviorMetadata":
             equipment=equipment,
             stimulus_frame_rate=stimulus_frame_rate,
             session_type=session_type,
-            behavior_session_uuid=session_uuid
+            behavior_session_uuid=session_uuid,
+            project_code=project_code,
         )
 
     @property
@@ -321,6 +334,10 @@ def behavior_session_uuid(self) -> Optional[uuid.UUID]:
     def behavior_session_id(self) -> int:
         return self._behavior_session_id.value
 
+    @property
+    def project_code(self) -> str:
+        return self._project_code.value
+
     @property
     def subject_metadata(self):
         return self._subject_metadata
@@ -344,7 +361,8 @@ def to_nwb(self, nwbfile: NWBFile) -> NWBFile:
             behavior_session_uuid=str(self.behavior_session_uuid),
             stimulus_frame_rate=self.stimulus_frame_rate,
             session_type=self.session_type,
-            equipment_name=self.equipment.value
+            equipment_name=self.equipment.value,
+            project_code=self.project_code
         )
         nwbfile.add_lab_meta_data(nwb_metadata)
 

allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/project_code.py

@@ -0,0 +1,42 @@
+from pynwb import NWBFile
+from typing import Optional
+
+from allensdk.core import DataObject
+from allensdk.core import \
+    LimsReadableInterface
+from allensdk.internal.api import PostgresQueryMixin
+from allensdk.core import NwbReadableInterface
+
+
+class ProjectCode(DataObject, LimsReadableInterface, NwbReadableInterface):
+    """Unique identifier for the project this BehaviorSession is associated
+    with. Project ids can be used internally to extract a project code.
+
+    If project_Code is null/None, we set the value to string 'Not Available'.
+    """
+
+    def __init__(self, project_code: Optional[str] = None):
+        if project_code is None:
+            project_code = 'Not Available'
+        super().__init__(name='project_code', value=project_code)
+
+    @classmethod
+    def from_lims(cls, behavior_session_id: int,
+                  lims_db: PostgresQueryMixin) -> "ProjectCode":
+        query = f"""
+            SELECT ps.code AS project_code
+            FROM behavior_sessions bs
+            LEFT JOIN projects ps on bs.project_id = ps.id
+            WHERE bs.id = {behavior_session_id}
+        """
+        project_code = lims_db.fetchone(query, strict=False)
+        return cls(project_code=project_code)
+
+    @classmethod
+    def from_nwb(cls, nwbfile: NWBFile) -> "ProjectCode":
+        try:
+            metadata = nwbfile.lab_meta_data['metadata']
+            return cls(project_code=metadata.project_code)
+        except AttributeError:
+            # Return values for NWBs without the project code set/available.
+            return cls()

allensdk/brain_observatory/behavior/data_objects/metadata/behavior_ophys_metadata.py

@@ -1,4 +1,5 @@
 from typing import Union
+import warnings
 
 from pynwb import NWBFile
 
@@ -60,6 +61,13 @@ def from_lims(cls, ophys_experiment_id: int,
             ophys_metadata = OphysExperimentMetadata.from_lims(
                 ophys_experiment_id=ophys_experiment_id, lims_db=lims_db)
 
+        if ophys_metadata.project_code != behavior_metadata.project_code:
+            raise warnings.warn(
+                'project_code for Ophys experiment table does not match '
+                'project_code from behavior_session table for '
+                'ophys_experiment_id={ophys_experiment_id} with '
+                f'behavior_session_id={behavior_session_id}.')
+
         return cls(behavior_metadata=behavior_metadata,
                    ophys_metadata=ophys_metadata)
 
@@ -147,7 +155,8 @@ def to_nwb(self, nwbfile: NWBFile) -> NWBFile:
             imaging_depth=ophys_meta.imaging_depth,
             targeted_imaging_depth=ophys_meta.targeted_imaging_depth,
             behavior_session_uuid=str(behavior_meta.behavior_session_uuid),
-            behavior_session_id=behavior_meta.behavior_session_id
+            behavior_session_id=behavior_meta.behavior_session_id,
+            project_code=ophys_meta.project_code,
         )
         nwbfile.add_lab_meta_data(nwb_metadata)
 

allensdk/brain_observatory/behavior/data_objects/metadata/ophys_experiment_metadata/multi_plane_metadata/multi_plane_metadata.py

@@ -6,7 +6,7 @@
 from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.ophys_container_id import OphysContainerId  # NOQA
 from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.ophys_experiment_metadata import OphysExperimentMetadata  # NOQA
 from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.ophys_session_id import OphysSessionId  # NOQA
-from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.project_code import ProjectCode  # NOQA
+from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.ophys_project_code import OphysProjectCode  # NOQA
 from allensdk.brain_observatory.behavior.data_objects.metadata.ophys_experiment_metadata.targeted_imaging_depth import TargetedImagingDepth  # NOQA
 from allensdk.internal.api import PostgresQueryMixin
 
@@ -20,7 +20,7 @@ def __init__(self,
                  imaging_depth: ImagingDepth,
                  targeted_imaging_depth: TargetedImagingDepth,
                  imaging_plane_group: ImagingPlaneGroup,
-                 project_code: ProjectCode):
+                 project_code: OphysProjectCode = OphysProjectCode()):
         super().__init__(
             ophys_experiment_id=ophys_experiment_id,
             ophys_session_id=ophys_session_id,