From 80c066cb1e5c70cedcd37aa8f598bbdf557692ac Mon Sep 17 00:00:00 2001 From: Matthew Richards <32678030+MRichards99@users.noreply.github.com> Date: Mon, 24 Jan 2022 16:56:05 +0000 Subject: [PATCH 1/5] refactor: update PaNOSC mappings Co-authored-by: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> --- datagateway_api/search_api_mapping.json.example | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/datagateway_api/search_api_mapping.json.example b/datagateway_api/search_api_mapping.json.example index 57161585..ebef5f30 100644 --- a/datagateway_api/search_api_mapping.json.example +++ b/datagateway_api/search_api_mapping.json.example @@ -63,14 +63,15 @@ "affiliation": {"Affiliation": "user.dataPublicationUsers.affiliations"} }, "Parameter": { - "base_icat_entity": "InvestigationParameter", + "base_icat_entity": ["InvestigationParameter", "DatasetParameter"], "id": "id", - "name": "name", + "name": "type.name", "value": ["numericValue", "stringValue", "dateTimeValue"], "unit": "type.units", - "dataset": {"Dataset": "investigation.investigationInstruments.instrument.datasetInstruments.dataset"}, + "dataset": {"Dataset": ["investigation.investigationInstruments.instrument.datasetInstruments.dataset", "dataset"]}, "document": {"Document": "investigation"} }, + }, "Person": { "base_icat_entity": "User", "id": "id", From f61ed00c69853e4b4d00d5d610c93dcd5aa9057a Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 26 Jan 2022 11:43:16 +0000 Subject: [PATCH 2/5] refactor: make changes for new Python ICAT implementation #259 --- datagateway_api/src/search_api/filters.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/datagateway_api/src/search_api/filters.py b/datagateway_api/src/search_api/filters.py index 0b5fa9d2..85c2988a 100644 --- a/datagateway_api/src/search_api/filters.py +++ b/datagateway_api/src/search_api/filters.py @@ -1,4 +1,3 @@ -from icat.client import Client from icat.query import Query from datagateway_api.src.datagateway_api.icat.filters import ( @@ -7,6 +6,7 @@ PythonICATSkipFilter, PythonICATWhereFilter, ) +from datagateway_api.src.search_api.session_handler import SessionHandler # TODO - Implement each of these filters for Search API, inheriting from the Python ICAT # versions @@ -20,17 +20,14 @@ def apply_filter(self, query): return super().apply_filter(query) def __str__(self): - # TODO - replace with `SessionHandler.client` when that work is merged - client = Client("https://localhost.localdomain:8181", checkCert=False) - client.login("simple", {"username": "root", "password": "pw"}) - # TODO - can't just hardcode investigation entity. Might need `icat_entity_name` # to be passed into init - query = Query(client, "Investigation") + query = Query(SessionHandler.client, "Investigation") query.addConditions(self.create_filter()) - str_conds = query.get_conditions_as_str() + str_conds = query.where_clause + str_conds = str_conds.replace("WHERE ", "") - return str_conds[0] + return str_conds def __repr__(self): return ( From 2478ffafee4b5ee4854d24b224c4c812ebd6c179 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 26 Jan 2022 11:53:19 +0000 Subject: [PATCH 3/5] refactor: implement new Python ICAT changes for read-only queries #260 - This removes lots of the complexities of the previous way to build complex WHERE clauses --- .../src/search_api/condition_setting_query.py | 42 +++++++++++++++++++ datagateway_api/src/search_api/filters.py | 36 ++++++++-------- datagateway_api/src/search_api/query.py | 39 ++++++++++++++++- 3 files changed, 97 insertions(+), 20 deletions(-) create mode 100644 datagateway_api/src/search_api/condition_setting_query.py diff --git a/datagateway_api/src/search_api/condition_setting_query.py b/datagateway_api/src/search_api/condition_setting_query.py new file mode 100644 index 00000000..efdf779a --- /dev/null +++ b/datagateway_api/src/search_api/condition_setting_query.py @@ -0,0 +1,42 @@ +from icat.query import Query + + +class ConditionSettingQuery(Query): + """ + Custom Query class to support the getting and setting of WHERE clauses outside of + the typical `Query.conditions` dict + """ + + def __init__( + self, + client, + entity_name, + conditions=None, + aggregate=None, + includes=None, + str_conditions="", + ): + + super().__init__( + client, + entity_name, + conditions=conditions, + aggregate=aggregate, + includes=includes, + ) + self.setConditionsByString(str_conditions) + + def setConditionsByString(self, str_conditions): # noqa: N802 + self._str_conditions = str_conditions + + @property + def where_clause(self): + """ + Overriding Python ICAT's implementation to support the creation of WHERE clauses + within the search API + """ + + if self._str_conditions: + return f"WHERE {self._str_conditions}" + else: + return super().where_clause diff --git a/datagateway_api/src/search_api/filters.py b/datagateway_api/src/search_api/filters.py index 7a75ecf1..24c1ef43 100644 --- a/datagateway_api/src/search_api/filters.py +++ b/datagateway_api/src/search_api/filters.py @@ -1,8 +1,8 @@ +from copy import copy from datetime import datetime import logging from datagateway_api.src.common.date_handler import DateHandler -from datagateway_api.src.common.exceptions import FilterError from datagateway_api.src.datagateway_api.icat.filters import ( PythonICATIncludeFilter, PythonICATLimitFilter, @@ -129,27 +129,25 @@ def __str__(self): """ if isinstance(self.search_api_query, SearchAPIQuery): - log.info("__str__ for SearchAPIWhereFilter, SearchAPIQuery found") + # Making a copy of the filter because `apply_filter()` can only be executed + # once per filter successfully + filter_copy = copy(self) + # Applying filter to the query so we get the correct JOINs, something not + # managed by the search API when we build the WHERE clause ourselves self.apply_filter(self.search_api_query) - # Replicating the condition in Python ICAT format so it can be searched on - # the query and return as string representation - conds_dict = self.create_filter() - a, jpql_func = self.search_api_query.icat_query.query._split_db_functs( - self.field, - ) - conds_dict[self.field] = self.search_api_query.icat_query.query._cond_value( - conds_dict[self.field], jpql_func, - ) - - str_conds = self.search_api_query.icat_query.query.search_conditions( - self.field, conds_dict, - ) - try: - return str_conds[0] - except IndexError: - raise FilterError("Condition could not be found in Python ICAT query") + # Using a blank query to ensure the correct condition is retrieved from the + # where clause + blank_query = SearchAPIQuery(self.search_api_query.panosc_entity_name) + filter_copy.apply_filter(blank_query) + where_clause = blank_query.icat_query.query.where_clause + # The WHERE keyword is added by `ConditionSettingQuery`, where the + # `where_clause` property is overridden to support the search API building + # its own WHERE clauses to support `NestedWhereFilters` + str_cond = where_clause.replace("WHERE ", "") + + return str_cond else: log.info( "__str__ for SearchAPIWhereFilter, no query found so repr() will be" diff --git a/datagateway_api/src/search_api/query.py b/datagateway_api/src/search_api/query.py index ecf4d95a..c7c882c2 100644 --- a/datagateway_api/src/search_api/query.py +++ b/datagateway_api/src/search_api/query.py @@ -1,4 +1,6 @@ +from datagateway_api.src.common.exceptions import SearchAPIError from datagateway_api.src.datagateway_api.icat.query import ICATQuery +from datagateway_api.src.search_api.condition_setting_query import ConditionSettingQuery from datagateway_api.src.search_api.panosc_mappings import mappings from datagateway_api.src.search_api.session_handler import SessionHandler @@ -10,10 +12,45 @@ def __init__(self, panosc_entity_name): "base_icat_entity" ] - self.icat_query = ICATQuery(SessionHandler.client, self.icat_entity_name) + self.icat_query = SearchAPIICATQuery( + SessionHandler.client, self.icat_entity_name, + ) def __repr__(self): return ( f"PaNOSC Entity Name: {self.panosc_entity_name}, ICAT Entity Name:" f" {self.icat_entity_name}, ICAT Query: {str(self.icat_query.query)}" ) + + +class SearchAPIICATQuery(ICATQuery): + """ + Class which has identical functionality to `ICATQuery` but uses a different + `__init__` to call `ConditionSettingQuery` instead of the base `Query`. That class + contains features required for the search API + """ + + def __init__( + self, + client, + entity_name, + conditions=None, + aggregate=None, + includes=None, + str_conditions=None, + ): + try: + self.query = ConditionSettingQuery( + client, + entity_name, + conditions=conditions, + aggregate=aggregate, + includes=includes, + str_conditions=str_conditions, + ) + + self.query.manual_count = False + except ValueError as e: + raise SearchAPIError( + f"An issue has occurred while creating a query for ICAT: {e}", + ) From c13c0db07a1a23a662d5ce32d7fb2f4709bc282d Mon Sep 17 00:00:00 2001 From: Matthew Richards <32678030+MRichards99@users.noreply.github.com> Date: Mon, 24 Jan 2022 16:56:05 +0000 Subject: [PATCH 4/5] refactor: update PaNOSC mappings Co-authored-by: Viktor Bozhinov <45173816+VKTB@users.noreply.github.com> --- datagateway_api/search_api_mapping.json.example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datagateway_api/search_api_mapping.json.example b/datagateway_api/search_api_mapping.json.example index 57161585..f93c2a04 100644 --- a/datagateway_api/search_api_mapping.json.example +++ b/datagateway_api/search_api_mapping.json.example @@ -63,12 +63,12 @@ "affiliation": {"Affiliation": "user.dataPublicationUsers.affiliations"} }, "Parameter": { - "base_icat_entity": "InvestigationParameter", + "base_icat_entity": ["InvestigationParameter", "DatasetParameter"], "id": "id", - "name": "name", + "name": "type.name", "value": ["numericValue", "stringValue", "dateTimeValue"], "unit": "type.units", - "dataset": {"Dataset": "investigation.investigationInstruments.instrument.datasetInstruments.dataset"}, + "dataset": {"Dataset": ["investigation.investigationInstruments.instrument.datasetInstruments.dataset", "dataset"]}, "document": {"Document": "investigation"} }, "Person": { From 3802cc9ee71a355d0ad87529f65112e8c3f8b881 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 26 Jan 2022 12:38:09 +0000 Subject: [PATCH 5/5] fix: fix example mapping file --- datagateway_api/search_api_mapping.json.example | 1 - 1 file changed, 1 deletion(-) diff --git a/datagateway_api/search_api_mapping.json.example b/datagateway_api/search_api_mapping.json.example index ebef5f30..f93c2a04 100644 --- a/datagateway_api/search_api_mapping.json.example +++ b/datagateway_api/search_api_mapping.json.example @@ -71,7 +71,6 @@ "dataset": {"Dataset": ["investigation.investigationInstruments.instrument.datasetInstruments.dataset", "dataset"]}, "document": {"Document": "investigation"} }, - }, "Person": { "base_icat_entity": "User", "id": "id",