Merge pull request #160 from ral-facilities/feature/icat-limit-skip-f…

…ilters-#139

Implement Skip/Limit Filters for Python ICAT Backend

common/config.py

@@ -1,6 +1,10 @@
 import json
 import sys
 from pathlib import Path
+import requests
+import logging
+
+log = logging.getLogger()
 
 
 class Config(object):
@@ -68,5 +72,17 @@ def get_port(self):
         except:
             sys.exit("Missing config value, port")
 
+    def get_icat_properties(self):
+        """
+        ICAT properties can be retrieved using Python ICAT's client object, however this
+        requires the client object to be authenticated which may not always be the case
+        when requesting these properties, hence a HTTP request is sent as an alternative
+        """
+        properties_url = f"{config.get_icat_url()}/icat/properties"
+        r = requests.request("GET", properties_url, verify=config.get_icat_check_cert())
+        icat_properties = r.json()
+
+        return icat_properties
+
 
 config = Config()

common/constants.py

@@ -4,3 +4,4 @@
 class Constants:
     DATABASE_URL = config.get_db_url()
     ACCEPTED_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+    ICAT_PROPERTIES = config.get_icat_properties()

common/filter_order_handler.py

@@ -13,6 +13,9 @@ def add_filter(self, filter):
     def add_filters(self, filters):
         self.filters.extend(filters)
 
+    def remove_filter(self, filter):
+        self.filters.remove(filter)
+
     def sort_filters(self):
         """
         Sorts the filters according to the order of operations

common/icat/filters.py

@@ -9,6 +9,8 @@
     IncludeFilter,
 )
 from common.exceptions import FilterError
+from common.config import config
+from common.constants import Constants
 
 log = logging.getLogger()
 
@@ -103,15 +105,37 @@ def __init__(self, skip_value):
         super().__init__(skip_value)
 
     def apply_filter(self, query):
-        pass
+        icat_properties = Constants.ICAT_PROPERTIES
+        icat_set_limit(query, self.skip_value, icat_properties["maxEntities"])
 
 
 class PythonICATLimitFilter(LimitFilter):
     def __init__(self, limit_value):
         super().__init__(limit_value)
+        self.skip_value = 0
 
     def apply_filter(self, query):
-        pass
+        icat_set_limit(query, self.skip_value, self.limit_value)
+
+
+def icat_set_limit(query, skip_number, limit_number):
+    """
+    Add limit (utilising skip and count) to an ICAT query
+
+    :param query: ICAT Query object to execute within Python ICAT
+    :type query: :class:`icat.query.Query`
+    :param skip_number: Number of results to skip
+    :type skip_number: :class:`int`
+    :param limit_number: Number of results to limit in the query
+    :type limit_number: :class:`int`
+    :raises FilterError: If the tuple is not of two elements, or the elements aren't of
+        the valid type
+    """
+    try:
+        query.setLimit((skip_number, limit_number))
+    except TypeError as e:
+        # Not a two element tuple as managed by Python ICAT's setLimit()
+        raise FilterError(e)
 
 
 class PythonICATIncludeFilter(IncludeFilter):

common/icat/helpers.py

@@ -12,7 +12,12 @@
 )
 from common.filter_order_handler import FilterOrderHandler
 from common.constants import Constants
-from common.icat.filters import PythonICATOrderFilter, PythonICATWhereFilter
+from common.icat.filters import (
+    PythonICATLimitFilter,
+    PythonICATWhereFilter,
+    PythonICATSkipFilter,
+    PythonICATOrderFilter,
+)
 
 
 log = logging.getLogger()
@@ -378,6 +383,7 @@ def get_entity_with_filters(client, table_name, filters):
 
     filter_handler = FilterOrderHandler()
     filter_handler.add_filters(filters)
+    merge_limit_skip_filters(filter_handler)
     clear_order_filters(filter_handler.filters)
     filter_handler.apply_filters(query)
 
@@ -389,6 +395,37 @@ def get_entity_with_filters(client, table_name, filters):
         return data
 
 
+def merge_limit_skip_filters(filter_handler):
+    """
+    When there are both limit and skip filters in a request, merge them into the limit
+    filter and remove the skip filter from `filter_handler`
+
+    :param filter_handler: The filter handler to apply the filters
+    :param filters: The filters to be applied
+    """
+
+    if any(
+        isinstance(filter, PythonICATSkipFilter) for filter in filter_handler.filters
+    ) and any(
+        isinstance(filter, PythonICATLimitFilter) for filter in filter_handler.filters
+    ):
+        # Merge skip and limit filter into a single limit filter
+        for filter in filter_handler.filters:
+            if isinstance(filter, PythonICATSkipFilter):
+                skip_filter = filter
+                request_skip_value = filter.skip_value
+
+            if isinstance(filter, PythonICATLimitFilter):
+                limit_filter = filter
+
+        if skip_filter and limit_filter:
+            log.info("Merging skip filter with limit filter")
+            limit_filter.skip_value = skip_filter.skip_value
+            log.info("Removing skip filter from list of filters")
+            filter_handler.remove_filter(skip_filter)
+            log.debug("Filters: %s", filter_handler.filters)
+
+
 def clear_order_filters(filters):
     """
     Checks if any order filters have been added to the request and resets the variable