Merge pull request #168 from ral-facilities/feature/icat-include-filt…

…er-#143

Implement Include Filter for Python ICAT Backend

common/database/filters.py

@@ -6,7 +6,8 @@
     LimitFilter,
     IncludeFilter,
 )
-from common.exceptions import FilterError
+from common.exceptions import FilterError, MultipleIncludeError
+from common.models import db_models
 
 from sqlalchemy import asc, desc
 

common/icat/filters.py

@@ -160,7 +160,70 @@ def icat_set_limit(query, skip_number, limit_number):
 
 class PythonICATIncludeFilter(IncludeFilter):
     def __init__(self, included_filters):
-        super().__init__(included_filters)
+        self.included_filters = []
+        log.info("Extracting fields for include filter")
+        self._extract_filter_fields(included_filters["include"])
+
+    def _extract_filter_fields(self, field):
+        """
+        Using recursion, go through the fields and add them to the filter's instance.
+        This means that lists within dictionaries, dictionaries within dictionaries are
+        supported. Where dictionaries are involved, '.' are used to join the fields
+        together
+
+        Some (but not all) fields require the plural to be accepted in the include of a
+        Python ICAT query - e.g. 'userGroups' is valid (plural required), but 'dataset'
+        is also valid (plural not required). The dictionary `substnames` in Python
+        ICAT's query.py gives a good overview of which need to be plural.
+
+        :param field: Which field(s) should be included in the ICAT query
+        :type field: :class:`str` or :class:`list` or :class:`dict`
+        """
+        if isinstance(field, str):
+            self.included_filters.append(field)
+        elif isinstance(field, dict):
+            for key, value in field.items():
+                if not isinstance(key, str):
+                    raise FilterError(
+                        "Include Filter: Dictionary key should only be a string, not"
+                        " any other type"
+                    )
+
+                if isinstance(value, str):
+                    self._extract_filter_fields(".".join((key, value)))
+                elif isinstance(value, list):
+                    for element in value:
+                        # Will end up as: key.element1, key.element2, key.element3 etc.
+                        self._extract_filter_fields(".".join((key, element)))
+                elif isinstance(value, dict):
+                    for inner_key, inner_value in value.items():
+                        if not isinstance(inner_key, str):
+                            raise FilterError(
+                                "Include Filter: Dictionary key should only be a string"
+                                ", not any other type"
+                            )
+
+                        # Will end up as: key.inner_key.inner_value
+                        self._extract_filter_fields(
+                            {".".join((key, inner_key)): inner_value}
+                        )
+                else:
+                    raise FilterError(
+                        "Include Filter: Inner field type (inside dictionary) not"
+                        " recognised, cannot interpret input"
+                    )
+        elif isinstance(field, list):
+            for element in field:
+                self._extract_filter_fields(element)
+        else:
+            raise FilterError(
+                "Include Filter: Field type not recognised, cannot interpret input"
+            )
 
     def apply_filter(self, query):
-        pass
+        log.info("Applying include filter, adding fields: %s", self.included_filters)
+
+        try:
+            query.addIncludes(self.included_filters)
+        except ValueError as e:
+            raise FilterError(e)

common/icat/helpers.py

@@ -190,7 +190,8 @@ def update_attributes(old_entity, new_entity):
     Python ICAT
 
     :param old_entity: An existing entity record from Python ICAT
-    :type object: :class:`icat.entities.ENTITY`
+    :type object: :class:`icat.entities.ENTITY` (implementation of
+        :class:`icat.entity.Entity`)
     :param new_entity: Dictionary containing the new data to be modified
     :type new_entity: :class:`dict`
     :raises BadRequestError: If the attribute cannot be found, or if it cannot be edited

common/icat/query.py

@@ -1,6 +1,7 @@
 import logging
 from datetime import datetime
 
+from icat.entity import Entity, EntityList
 from icat.query import Query
 from icat.exception import ICATValidationError
 
@@ -36,6 +37,7 @@ def __init__(
         """
 
         try:
+            log.info("Creating ICATQuery for entity: %s", entity_name)
             self.query = Query(
                 client,
                 entity_name,
@@ -63,18 +65,20 @@ def execute_query(self, client, return_json_formattable=False):
             manipulated at some point)
         :type return_json_formattable_data: :class:`bool`
         :return: Data (of type list) from the executed query
+        :raises PythonICATError: If an error occurs during query execution
         """
 
         try:
+            log.debug("Executing ICAT query")
             query_result = client.search(self.query)
         except ICATValidationError as e:
             raise PythonICATError(e)
 
         if self.query.aggregate == "DISTINCT":
+            log.info("Extracting the distinct fields from query's conditions")
             distinct_filter_flag = True
             # Check query's conditions for the ones created by the distinct filter
             self.attribute_names = []
-            log.debug("Query conditions: %s", self.query.conditions)
 
             for key, value in self.query.conditions.items():
                 # Value can be a list if there's multiple WHERE filters for the same
@@ -93,36 +97,27 @@ def execute_query(self, client, return_json_formattable=False):
             distinct_filter_flag = False
 
         if return_json_formattable:
+            log.info("Query results will be returned in a JSON format")
             data = []
+
             for result in query_result:
-                dict_result = result.as_dict()
                 distinct_result = {}
+                dict_result = self.entity_to_dict(result, self.query.includes)
 
-                for key in dict_result:
-                    # Convert datetime objects to strings so they can be JSON
-                    # serialisable
-                    if isinstance(dict_result[key], datetime):
-                        # Remove timezone data which isn't utilised in ICAT
-                        dict_result[key] = (
-                            dict_result[key]
-                            .replace(tzinfo=None)
-                            .strftime(Constants.ACCEPTED_DATE_FORMAT)
-                        )
-
+                for key, value in dict_result.items():
                     if distinct_filter_flag:
                         # Add only the required data as per request's distinct filter
                         # fields
                         if key in self.attribute_names:
                             distinct_result[key] = dict_result[key]
 
-                # Add to the response's data depending on whether request has a distinct
-                # filter
                 if distinct_filter_flag:
                     data.append(distinct_result)
                 else:
                     data.append(dict_result)
             return data
         else:
+            log.info("Query results will be returned as ICAT entities")
             return query_result
 
     def check_attribute_name_for_distinct(self, key, value):
@@ -139,3 +134,71 @@ def check_attribute_name_for_distinct(self, key, value):
         """
         if value == Constants.PYTHON_ICAT_DISTNCT_CONDITION:
             self.attribute_names.append(key)
+
+    def datetime_object_to_str(self, date_obj):
+        """
+        Convert a datetime object to a string so it can be outputted in JSON
+
+        There's currently no reason to make this function static, but it could be useful
+        in the future if a use case required this functionality.
+
+        :param date_obj: Datetime object from data from an ICAT entity
+        :type date_obj: :class:`datetime.datetime`
+        :return: Datetime (of type string) in the agreed format
+        """
+        return date_obj.replace(tzinfo=None).strftime(Constants.ACCEPTED_DATE_FORMAT)
+
+    def entity_to_dict(self, entity, includes):
+        """
+        This expands on Python ICAT's implementation of `icat.entity.Entity.as_dict()`
+        to use set operators to create a version of the entity as a dictionary
+
+        Most of this function is dedicated to recursing over included fields from a
+        query, since this is functionality isn't part of Python ICAT's `as_dict()`. This
+        function can be used when there are no include filters in the query/request
+        however.
+
+        :param entity: Python ICAT entity from an ICAT query
+        :type entity: :class:`icat.entities.ENTITY` (implementation of
+            :class:`icat.entity.Entity`) or :class:`icat.entity.EntityList`
+        :param includes: Set of fields that have been included in the ICAT query. Where
+            fields have a chain of relationships, they're a single element string
+            separated by dots
+        :type includes: :class:`set`
+        :return: ICAT Data (of type dictionary) ready to be serialised to JSON
+        """
+        d = {}
+
+        # Split up the fields separated by dots and flatten the resulting lists
+        flat_includes = [m for n in (field.split(".") for field in includes) for m in n]
+
+        # Verifying that `flat_includes` only has fields which are related to the entity
+        include_set = (entity.InstRel | entity.InstMRel) & set(flat_includes)
+        for key in entity.InstAttr | entity.MetaAttr | include_set:
+            if key in flat_includes:
+                target = getattr(entity, key)
+                # Copy and remove don't return values so must be done separately
+                includes_copy = flat_includes.copy()
+                try:
+                    includes_copy.remove(key)
+                except ValueError:
+                    log.warning(
+                        "Key couldn't be found to remove from include list, this could"
+                        " cause an issue further on in the request"
+                    )
+                if isinstance(target, Entity):
+                    d[key] = self.entity_to_dict(target, includes_copy)
+                # Related fields with one-many relationships are stored as EntityLists
+                elif isinstance(target, EntityList):
+                    d[key] = []
+                    for e in target:
+                        d[key].append(self.entity_to_dict(e, includes_copy))
+            # Add actual piece of data to the dictionary
+            else:
+                entity_data = getattr(entity, key)
+                # Convert datetime objects to strings ready to be outputted as JSON
+                if isinstance(entity_data, datetime):
+                    # Remove timezone data which isn't utilised in ICAT
+                    entity_data = self.datetime_object_to_str(entity_data)
+                d[key] = entity_data
+        return d