Merge pull request #171 from ral-facilities/feature/remaining-icat-en…

…dpoints-#145

Implement Remaining Standard Endpoints for Python ICAT Backend

common/backend.py

@@ -142,7 +142,7 @@ def update_with_id(self, session_id, entity_type, id_, data):
         pass
 
     @abstractmethod
-    def get_instrument_facilitycycles_with_filters(
+    def get_facility_cycles_for_instrument_with_filters(
         self, session_id, instrument_id, filters
     ):
         """
@@ -157,7 +157,7 @@ def get_instrument_facilitycycles_with_filters(
         pass
 
     @abstractmethod
-    def count_instrument_facilitycycles_with_filters(
+    def get_facility_cycles_for_instrument_count_with_filters(
         self, session_id, instrument_id, filters
     ):
         """
@@ -172,7 +172,7 @@ def count_instrument_facilitycycles_with_filters(
         pass
 
     @abstractmethod
-    def get_instrument_facilitycycle_investigations_with_filters(
+    def get_investigations_for_instrument_in_facility_cycle_with_filters(
         self, session_id, instrument_id, facilitycycle_id, filters
     ):
         """
@@ -188,7 +188,7 @@ def get_instrument_facilitycycle_investigations_with_filters(
         pass
 
     @abstractmethod
-    def count_instrument_facilitycycles_investigations_with_filters(
+    def get_investigations_for_instrument_in_facility_cycle_count_with_filters(
         self, session_id, instrument_id, facilitycycle_id, filters
     ):
         """

common/constants.py

@@ -3,6 +3,5 @@
 
 class Constants:
     DATABASE_URL = config.get_db_url()
-    ACCEPTED_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
     PYTHON_ICAT_DISTNCT_CONDITION = "!= null"
     ICAT_PROPERTIES = config.get_icat_properties()

common/database/backend.py

@@ -101,21 +101,21 @@ def update_with_id(self, session_id, table, id_, data):
 
     @requires_session_id
     @queries_records
-    def get_instrument_facilitycycles_with_filters(
+    def get_facility_cycles_for_instrument_with_filters(
         self, session_id, instrument_id, filters
     ):
         return get_facility_cycles_for_instrument(instrument_id, filters)
 
     @requires_session_id
     @queries_records
-    def count_instrument_facilitycycles_with_filters(
+    def get_facility_cycles_for_instrument_count_with_filters(
         self, session_id, instrument_id, filters
     ):
         return get_facility_cycles_for_instrument_count(instrument_id, filters)
 
     @requires_session_id
     @queries_records
-    def get_instrument_facilitycycle_investigations_with_filters(
+    def get_investigations_for_instrument_in_facility_cycle_with_filters(
         self, session_id, instrument_id, facilitycycle_id, filters
     ):
         return get_investigations_for_instrument_in_facility_cycle(
@@ -124,7 +124,7 @@ def get_instrument_facilitycycle_investigations_with_filters(
 
     @requires_session_id
     @queries_records
-    def count_instrument_facilitycycles_investigations_with_filters(
+    def get_investigations_for_instrument_in_facility_cycle_count_with_filters(
         self, session_id, instrument_id, facilitycycle_id, filters
     ):
         return get_investigations_for_instrument_in_facility_cycle_count(

common/database/helpers.py

@@ -233,7 +233,7 @@ def get_query_filter(filter):
         elif filter_name == "limit":
             return LimitFilter(filter["limit"])
         elif filter_name == "include":
-            return IncludeFilter(filter)
+            return IncludeFilter(filter["include"])
         elif filter_name == "distinct":
             return DistinctFieldFilter(filter["distinct"])
         else:
@@ -426,14 +426,14 @@ def patch_entities(table, json_list):
             if key.upper() == "ID":
                 update_row_from_id(table, json_list[key], json_list)
                 result = get_row_by_id(table, json_list[key])
-                results.append(result)
+                results.append(result.to_dict())
     else:
         for entity in json_list:
             for key in entity:
                 if key.upper() == "ID":
                     update_row_from_id(table, entity[key], entity)
                     result = get_row_by_id(table, entity[key])
-                    results.append(result)
+                    results.append(result.to_dict())
     if len(results) == 0:
         raise BadRequestError(f" Bad request made, request: {json_list}")
 

common/date_handler.py

@@ -0,0 +1,70 @@
+from dateutil.parser import parse
+from icat import helper
+
+from common.exceptions import BadRequestError
+
+
+class DateHandler:
+    """
+    Utility class to deal with dates. Currently, this class converts dates between
+    strings and `datetime.datetime` objects as well as detecting whether a string is
+    likely to be a date.
+    """
+
+    @staticmethod
+    def is_str_a_date(potential_date):
+        """
+        This function identifies if a string contains a date. This function doesn't
+        detect which format the date is, just if there's a date or not.
+
+        :param potential_date: String data that could contain a date of any format
+        :type potential_date: :class:`str`
+        :return: Boolean to signify whether `potential_date` is a date or not
+        """
+
+        try:
+            # Disabled fuzzy to avoid picking up dates in things like descriptions etc.
+            parse(potential_date, fuzzy=False)
+            return True
+        except ValueError:
+            return False
+
+    @staticmethod
+    def str_to_datetime_object(data):
+        """
+        Convert a string to a `datetime.datetime` object. This is commonly used when
+        storing user input in ICAT (using the Python ICAT backend).
+
+        Python 3.7+ has support for `datetime.fromisoformat()` which would be a more
+        elegant solution to this conversion operation since dates are converted into ISO
+        format within this file, however, the production instance of this API is
+        typically built on Python 3.6, and it doesn't seem of enough value to mandate
+        3.7 for a single line of code. Instead, a helper function from `python-icat` is
+        used which does the conversion using `suds`. This will convert inputs in the ISO
+        format (i.e. the format which Python ICAT, and therefore DataGateway API outputs
+        data) but also allows for conversion of other "sensible" formats.
+
+        :param data: Single data value from the request body
+        :type data: Data type of the data as per user's request body, :class:`str` is
+            assumed
+        :return: Date converted into a :class:`datetime` object
+        :raises BadRequestError: If there is an issue with the date format
+        """
+
+        try:
+            datetime_obj = helper.parse_attr_string(data, "Date")
+        except ValueError as e:
+            raise BadRequestError(e)
+
+        return datetime_obj
+
+    @staticmethod
+    def datetime_object_to_str(datetime_obj):
+        """
+        Convert a datetime object to a string so it can be outputted in JSON
+
+        :param datetime_obj: Datetime object from data from an ICAT entity
+        :type datetime_obj: :class:`datetime.datetime`
+        :return: Datetime (of type string) in the agreed format
+        """
+        return datetime_obj.isoformat(" ")

common/filter_order_handler.py

@@ -1,3 +1,14 @@
+import logging
+
+from common.icat.filters import (
+    PythonICATLimitFilter,
+    PythonICATSkipFilter,
+    PythonICATOrderFilter,
+)
+
+log = logging.getLogger()
+
+
 class FilterOrderHandler(object):
     """
     The FilterOrderHandler takes in filters, sorts them according to the order of
@@ -32,3 +43,65 @@ def apply_filters(self, query):
 
         for filter in self.filters:
             filter.apply_filter(query)
+
+    def merge_python_icat_limit_skip_filters(self):
+        """
+        When there are both limit and skip filters in a request, merge them into the
+        limit filter and remove the skip filter from the instance
+        """
+
+        if any(
+            isinstance(icat_filter, PythonICATSkipFilter)
+            for icat_filter in self.filters
+        ) and any(
+            isinstance(icat_filter, PythonICATLimitFilter)
+            for icat_filter in self.filters
+        ):
+            # Merge skip and limit filter into a single limit filter
+            for icat_filter in self.filters:
+                if isinstance(icat_filter, PythonICATSkipFilter):
+                    skip_filter = icat_filter
+                    request_skip_value = icat_filter.skip_value
+
+                if isinstance(icat_filter, PythonICATLimitFilter):
+                    limit_filter = icat_filter
+
+            if skip_filter and limit_filter:
+                log.info("Merging skip filter with limit filter")
+                limit_filter.skip_value = skip_filter.skip_value
+                log.info("Removing skip filter from list of filters")
+                self.remove_filter(skip_filter)
+                log.debug("Filters: %s", self.filters)
+
+    def clear_python_icat_order_filters(self):
+        """
+        Checks if any order filters have been added to the request and resets the
+        variable used to manage which attribute(s) to use for sorting results.
+        
+        A reset is required because Python ICAT overwrites (as opposed to appending to
+        it) the query's order list every time one is added to the query.
+        """
+
+        if any(
+            isinstance(icat_filter, PythonICATOrderFilter)
+            for icat_filter in self.filters
+        ):
+            PythonICATOrderFilter.result_order = []
+
+    def manage_icat_filters(self, filters, query):
+        """
+        Utility function to call other functions in this class, used to manage filters
+        when using the Python ICAT backend. These steps are the same with the different
+        types of requests that utilise filters, therefore this function helps to reduce
+        code duplication
+
+        :param filters: The list of filters that will be applied to the query
+        :type filters: List of specific implementations :class:`QueryFilter`
+        :param query: ICAT query which will fetch the data at a later stage
+        :type query: :class:`icat.query.Query`
+        """
+
+        self.add_filters(filters)
+        self.merge_python_icat_limit_skip_filters()
+        self.clear_python_icat_order_filters()
+        self.apply_filters(query)

common/filters.py

@@ -69,4 +69,4 @@ class IncludeFilter(QueryFilter):
     precedence = 5
 
     def __init__(self, included_filters):
-        self.included_filters = included_filters["include"]
+        self.included_filters = included_filters

common/icat/backend.py

@@ -15,6 +15,14 @@
     update_entity_by_id,
     delete_entity_by_id,
     get_entity_with_filters,
+    get_count_with_filters,
+    get_first_result_with_filters,
+    update_entities,
+    create_entities,
+    get_facility_cycles_for_instrument,
+    get_facility_cycles_for_instrument_count,
+    get_investigations_for_instrument_in_facility_cycle,
+    get_investigations_for_instrument_in_facility_cycle_count,
 )
 
 from common.config import config
@@ -72,21 +80,25 @@ def get_with_filters(self, session_id, table, filters, **kwargs):
     @queries_records
     def create(self, session_id, table, data, **kwargs):
         client = kwargs["client"] if kwargs["client"] else create_client()
+        return create_entities(client, table.__name__, data)
 
     @requires_session_id
     @queries_records
     def update(self, session_id, table, data, **kwargs):
         client = kwargs["client"] if kwargs["client"] else create_client()
+        return update_entities(client, table.__name__, data)
 
     @requires_session_id
     @queries_records
     def get_one_with_filters(self, session_id, table, filters, **kwargs):
         client = kwargs["client"] if kwargs["client"] else create_client()
+        return get_first_result_with_filters(client, table.__name__, filters)
 
     @requires_session_id
     @queries_records
     def count_with_filters(self, session_id, table, filters, **kwargs):
         client = kwargs["client"] if kwargs["client"] else create_client()
+        return get_count_with_filters(client, table.__name__, filters)
 
     @requires_session_id
     @queries_records
@@ -108,31 +120,36 @@ def update_with_id(self, session_id, table, id_, data, **kwargs):
 
     @requires_session_id
     @queries_records
-    def get_instrument_facilitycycles_with_filters(
-        self, session_id, instrument_id, filters, **kwargs
+    def get_facility_cycles_for_instrument_with_filters(
+        self, session_id, instrument_id, filters, **kwargs,
     ):
         client = kwargs["client"] if kwargs["client"] else create_client()
+        return get_facility_cycles_for_instrument(client, instrument_id, filters)
 
     @requires_session_id
     @queries_records
-    def count_instrument_facilitycycles_with_filters(
-        self, session_id, instrument_id, filters, **kwargs
+    def get_facility_cycles_for_instrument_count_with_filters(
+        self, session_id, instrument_id, filters, **kwargs,
     ):
         client = kwargs["client"] if kwargs["client"] else create_client()
-        # return get_facility_cycles_for_instrument_count(instrument_id, filters)
+        return get_facility_cycles_for_instrument_count(client, instrument_id, filters)
 
     @requires_session_id
     @queries_records
-    def get_instrument_facilitycycle_investigations_with_filters(
-        self, session_id, instrument_id, facilitycycle_id, filters, **kwargs
+    def get_investigations_for_instrument_in_facility_cycle_with_filters(
+        self, session_id, instrument_id, facilitycycle_id, filters, **kwargs,
     ):
         client = kwargs["client"] if kwargs["client"] else create_client()
-        # return get_investigations_for_instrument_in_facility_cycle(instrument_id, facilitycycle_id, filters)
+        return get_investigations_for_instrument_in_facility_cycle(
+            client, instrument_id, facilitycycle_id, filters
+        )
 
     @requires_session_id
     @queries_records
-    def count_instrument_facilitycycles_investigations_with_filters(
-        self, session_id, instrument_id, facilitycycle_id, filters, **kwargs
+    def get_investigations_for_instrument_in_facility_cycle_count_with_filters(
+        self, session_id, instrument_id, facilitycycle_id, filters, **kwargs,
     ):
         client = kwargs["client"] if kwargs["client"] else create_client()
-        # return get_investigations_for_instrument_in_facility_cycle_count(instrument_id, facilitycycle_id, filters)
+        return get_investigations_for_instrument_in_facility_cycle_count(
+            client, instrument_id, facilitycycle_id, filters
+        )

common/icat/filters.py

@@ -74,8 +74,11 @@ def create_condition(attribute_name, operator, value):
         # Removing quote marks when doing conditions with IN expressions or when a
         # distinct filter is used in a request
         jpql_value = (
-            f"{value}" if operator == "in" or operator == "!=" else f"'{value}'"
+            f"{value}"
+            if operator == "in" or operator == "!=" or "o." in str(value)
+            else f"'{value}'"
         )
+
         conditions[attribute_name] = f"{operator} {jpql_value}"
         log.debug("Conditions in ICAT where filter, %s", conditions)
         return conditions
@@ -88,7 +91,15 @@ def __init__(self, fields):
     def apply_filter(self, query):
         try:
             log.info("Adding ICAT distinct filter to ICAT query")
-            query.setAggregate("DISTINCT")
+            if (
+                query.aggregate == "COUNT"
+                or query.aggregate == "AVG"
+                or query.aggregate == "SUM"
+            ):
+                # Distinct can be combined with other aggregate functions
+                query.setAggregate(f"{query.aggregate}:DISTINCT")
+            else:
+                query.setAggregate("DISTINCT")
 
             # Using where filters to identify which fields to apply distinct too
             for field in self.fields:
@@ -163,7 +174,7 @@ class PythonICATIncludeFilter(IncludeFilter):
     def __init__(self, included_filters):
         self.included_filters = []
         log.info("Extracting fields for include filter")
-        self._extract_filter_fields(included_filters["include"])
+        self._extract_filter_fields(included_filters)
 
     def _extract_filter_fields(self, field):
         """