From 412cf1b73b095cec4d779e2f1bfb13fcaf95c17d Mon Sep 17 00:00:00 2001 From: Alejandra Gonzalez-Beltran Date: Fri, 17 Jul 2020 13:24:56 +0100 Subject: [PATCH 1/4] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 261eeb9e..c713f397 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2019 - Science and Technology Facilities Council – UK Research and Innovation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 7256627fd227213ca0a959c62c4d6258c3f4e191 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 14 Sep 2020 09:51:29 +0000 Subject: [PATCH 2/4] #139: Send a request for ICAT properties at start-up only --- common/config.py | 1 - common/constants.py | 1 + common/icat/filters.py | 3 ++- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/common/config.py b/common/config.py index 199c5fee..81e46e5c 100644 --- a/common/config.py +++ b/common/config.py @@ -81,7 +81,6 @@ def get_icat_properties(self): properties_url = f"{config.get_icat_url()}/icat/properties" r = requests.request("GET", properties_url, verify=config.get_icat_check_cert()) icat_properties = r.json() - log.debug("ICAT Properties: %s", icat_properties) return icat_properties diff --git a/common/constants.py b/common/constants.py index 9d3fef36..3c213a4f 100644 --- a/common/constants.py +++ b/common/constants.py @@ -4,3 +4,4 @@ class Constants: DATABASE_URL = config.get_db_url() ACCEPTED_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" + ICAT_PROPERTIES = config.get_icat_properties() diff --git a/common/icat/filters.py b/common/icat/filters.py index e54d861b..d990a797 100644 --- a/common/icat/filters.py +++ b/common/icat/filters.py @@ -10,6 +10,7 @@ ) from common.exceptions import FilterError from common.config import config +from common.constants import Constants log = logging.getLogger() @@ -104,7 +105,7 @@ def __init__(self, skip_value): super().__init__(skip_value) def apply_filter(self, query): - icat_properties = config.get_icat_properties() + icat_properties = Constants.ICAT_PROPERTIES icat_set_limit(query, self.skip_value, icat_properties["maxEntities"]) From 4f84e52847a7b3c7ceb72b32169f3e7449aa4aff Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 14 Sep 2020 10:05:19 +0000 Subject: [PATCH 3/4] #141: Rename ICATQuery and move it to a separate file --- common/icat/filters.py | 2 +- common/icat/helpers.py | 137 +-------------------------------------- common/icat/query.py | 141 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 135 deletions(-) create mode 100644 common/icat/query.py diff --git a/common/icat/filters.py b/common/icat/filters.py index 3dce4d53..8410a961 100644 --- a/common/icat/filters.py +++ b/common/icat/filters.py @@ -36,7 +36,7 @@ def apply_filter(self, query): where_filter = self.create_condition(self.field, ">=", self.value) elif self.operation == "in": # Convert self.value into a string with brackets equivalent to tuple format. - # Cannot convert straight to tuple as single element tuples contain a + # Cannot convert straight to tuple as single element tuples contain a # trailing comma which Python ICAT/JPQL doesn't accept self.value = str(self.value).replace("[", "(").replace("]", ")") where_filter = self.create_condition(self.field, "in", self.value) diff --git a/common/icat/helpers.py b/common/icat/helpers.py index adaa0a05..210d30ce 100644 --- a/common/icat/helpers.py +++ b/common/icat/helpers.py @@ -2,7 +2,6 @@ import logging from datetime import datetime, timedelta -from icat.query import Query from icat.exception import ICATSessionError, ICATValidationError from common.exceptions import ( AuthenticationError, @@ -18,6 +17,7 @@ PythonICATSkipFilter, PythonICATOrderFilter, ) +from common.icat.query import ICATQuery log = logging.getLogger() @@ -94,137 +94,6 @@ def refresh_client_session(client): client.refresh() -class icat_query: - def __init__( - self, client, entity_name, conditions=None, aggregate=None, includes=None - ): - """ - Create a Query object within Python ICAT - - :param client: ICAT client containing an authenticated user - :type client: :class:`icat.client.Client` - :param entity_name: Name of the entity to get data from - :type entity_name: :class:`suds.sax.text.Text` - :param conditions: Constraints used when an entity is queried - :type conditions: :class:`dict` - :param aggregate: Name of the aggregate function to apply. Operations such as - counting the number of records. See `icat.query.setAggregate` for valid - values. - :type aggregate: :class:`str` - :param includes: List of related entity names to add to the query so related - entities (and their data) can be returned with the query result - :type includes: :class:`str` or iterable of :class:`str` - :return: Query object from Python ICAT - :raises PythonICATError: If a ValueError is raised when creating a Query(), 500 - will be returned as a response - """ - - try: - self.query = Query( - client, - entity_name, - conditions=conditions, - aggregate=aggregate, - includes=includes, - ) - except ValueError: - raise PythonICATError( - "An issue has occurred while creating a Python ICAT Query object," - " suggesting an invalid argument" - ) - - def execute_query(self, client, return_json_formattable=False): - """ - Execute a previously created ICAT Query object and return in the format - specified by the return_json_formattable flag - - :param client: ICAT client containing an authenticated user - :type client: :class:`icat.client.Client` - :param return_json_formattable: Flag to determine whether the data from the - query should be returned as a list of data ready to be converted straight to - JSON (i.e. if the data will be used as a response for an API call) or - whether to leave the data in a Python ICAT format (i.e. if it's going to be - manipulated at some point) - :type return_json_formattable_data: :class:`bool` - :return: Data (of type list) from the executed query - """ - - try: - query_result = client.search(self.query) - except ICATValidationError as e: - raise PythonICATError(e) - - if self.query.aggregate == "DISTINCT": - distinct_filter_flag = True - # Check query's conditions for the ones created by the distinct filter - self.attribute_names = [] - log.debug("Query conditions: %s", self.query.conditions) - - for key, value in self.query.conditions.items(): - # Value can be a list if there's multiple WHERE filters for the same - # attribute name within an ICAT query - if isinstance(value, list): - for sub_value in value: - self.check_attribute_name_for_distinct(key, sub_value) - elif isinstance(value, str): - self.check_attribute_name_for_distinct(key, value) - log.debug( - "Attribute names used in the distinct filter, as captured by the" - " query's conditions %s", - self.attribute_names, - ) - else: - distinct_filter_flag = False - - if return_json_formattable: - data = [] - for result in query_result: - dict_result = result.as_dict() - distinct_result = {} - - for key in dict_result: - # Convert datetime objects to strings so they can be JSON - # serialisable - if isinstance(dict_result[key], datetime): - # Remove timezone data which isn't utilised in ICAT - dict_result[key] = ( - dict_result[key] - .replace(tzinfo=None) - .strftime(Constants.ACCEPTED_DATE_FORMAT) - ) - - if distinct_filter_flag: - # Add only the required data as per request's distinct filter - # fields - if key in self.attribute_names: - distinct_result[key] = dict_result[key] - - # Add to the response's data depending on whether request has a distinct - # filter - if distinct_filter_flag: - data.append(distinct_result) - else: - data.append(dict_result) - return data - else: - return query_result - - def check_attribute_name_for_distinct(self, key, value): - """ - Check the attribute name to see if its associated value is used to signify the - attribute is requested in a distinct filter and if so, append it to the list of - attribute names - - :param key: Name of an attribute - :type key: :class:`str` - :param value: Expression that should be applied to the associated attribute - e.g. "= 'Metadata'" - :type value: :class:`str` - """ - if value == Constants.PYTHON_ICAT_DISTNCT_CONDITION: - self.attribute_names.append(key) - - def get_python_icat_entity_name(client, database_table_name): """ From the database table name, this function returns the correctly cased entity name @@ -357,7 +226,7 @@ def get_entity_by_id(client, table_name, id_, return_json_formattable_data): # Set query condition for the selected ID id_condition = PythonICATWhereFilter.create_condition("id", "=", id_) - id_query = icat_query( + id_query = ICATQuery( client, selected_entity_name, conditions=id_condition, includes="1" ) entity_by_id_data = id_query.execute_query(client, return_json_formattable_data) @@ -426,7 +295,7 @@ def get_entity_with_filters(client, table_name, filters): """ selected_entity_name = get_python_icat_entity_name(client, table_name) - query = icat_query(client, selected_entity_name) + query = ICATQuery(client, selected_entity_name) filter_handler = FilterOrderHandler() filter_handler.add_filters(filters) diff --git a/common/icat/query.py b/common/icat/query.py new file mode 100644 index 00000000..e5411588 --- /dev/null +++ b/common/icat/query.py @@ -0,0 +1,141 @@ +import logging +from datetime import datetime + +from icat.query import Query +from icat.exception import ICATValidationError + +from common.exceptions import PythonICATError +from common.constants import Constants + +log = logging.getLogger() + + +class ICATQuery: + def __init__( + self, client, entity_name, conditions=None, aggregate=None, includes=None + ): + """ + Create a Query object within Python ICAT + + :param client: ICAT client containing an authenticated user + :type client: :class:`icat.client.Client` + :param entity_name: Name of the entity to get data from + :type entity_name: :class:`suds.sax.text.Text` + :param conditions: Constraints used when an entity is queried + :type conditions: :class:`dict` + :param aggregate: Name of the aggregate function to apply. Operations such as + counting the number of records. See `icat.query.setAggregate` for valid + values. + :type aggregate: :class:`str` + :param includes: List of related entity names to add to the query so related + entities (and their data) can be returned with the query result + :type includes: :class:`str` or iterable of :class:`str` + :return: Query object from Python ICAT + :raises PythonICATError: If a ValueError is raised when creating a Query(), 500 + will be returned as a response + """ + + try: + self.query = Query( + client, + entity_name, + conditions=conditions, + aggregate=aggregate, + includes=includes, + ) + except ValueError: + raise PythonICATError( + "An issue has occurred while creating a Python ICAT Query object," + " suggesting an invalid argument" + ) + + def execute_query(self, client, return_json_formattable=False): + """ + Execute a previously created ICAT Query object and return in the format + specified by the return_json_formattable flag + + :param client: ICAT client containing an authenticated user + :type client: :class:`icat.client.Client` + :param return_json_formattable: Flag to determine whether the data from the + query should be returned as a list of data ready to be converted straight to + JSON (i.e. if the data will be used as a response for an API call) or + whether to leave the data in a Python ICAT format (i.e. if it's going to be + manipulated at some point) + :type return_json_formattable_data: :class:`bool` + :return: Data (of type list) from the executed query + """ + + try: + query_result = client.search(self.query) + except ICATValidationError as e: + raise PythonICATError(e) + + if self.query.aggregate == "DISTINCT": + distinct_filter_flag = True + # Check query's conditions for the ones created by the distinct filter + self.attribute_names = [] + log.debug("Query conditions: %s", self.query.conditions) + + for key, value in self.query.conditions.items(): + # Value can be a list if there's multiple WHERE filters for the same + # attribute name within an ICAT query + if isinstance(value, list): + for sub_value in value: + self.check_attribute_name_for_distinct(key, sub_value) + elif isinstance(value, str): + self.check_attribute_name_for_distinct(key, value) + log.debug( + "Attribute names used in the distinct filter, as captured by the" + " query's conditions %s", + self.attribute_names, + ) + else: + distinct_filter_flag = False + + if return_json_formattable: + data = [] + for result in query_result: + dict_result = result.as_dict() + distinct_result = {} + + for key in dict_result: + # Convert datetime objects to strings so they can be JSON + # serialisable + if isinstance(dict_result[key], datetime): + # Remove timezone data which isn't utilised in ICAT + dict_result[key] = ( + dict_result[key] + .replace(tzinfo=None) + .strftime(Constants.ACCEPTED_DATE_FORMAT) + ) + + if distinct_filter_flag: + # Add only the required data as per request's distinct filter + # fields + if key in self.attribute_names: + distinct_result[key] = dict_result[key] + + # Add to the response's data depending on whether request has a distinct + # filter + if distinct_filter_flag: + data.append(distinct_result) + else: + data.append(dict_result) + return data + else: + return query_result + + def check_attribute_name_for_distinct(self, key, value): + """ + Check the attribute name to see if its associated value is used to signify the + attribute is requested in a distinct filter and if so, append it to the list of + attribute names + + :param key: Name of an attribute + :type key: :class:`str` + :param value: Expression that should be applied to the associated attribute + e.g. "= 'Metadata'" + :type value: :class:`str` + """ + if value == Constants.PYTHON_ICAT_DISTNCT_CONDITION: + self.attribute_names.append(key) From acc96727aa4f2be59b3336b8df44ab3871e702e4 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 29 Sep 2020 14:09:45 +0000 Subject: [PATCH 4/4] #141: Set LIKE operator on WHERE filter to do wildcard searches - This should allow these types of searches to provide more accurate results, where previously none could be found --- common/icat/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/icat/filters.py b/common/icat/filters.py index ab729d08..8420ef3a 100644 --- a/common/icat/filters.py +++ b/common/icat/filters.py @@ -26,7 +26,7 @@ def apply_filter(self, query): elif self.operation == "ne": where_filter = self.create_condition(self.field, "!=", self.value) elif self.operation == "like": - where_filter = self.create_condition(self.field, "like", self.value) + where_filter = self.create_condition(self.field, "like", f"%{self.value}%") elif self.operation == "lt": where_filter = self.create_condition(self.field, "<", self.value) elif self.operation == "lte":