From 34fa7d9896985da0bc3c1686cf40a28109d48b1f Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 13 Jan 2022 17:03:23 +0000 Subject: [PATCH 01/22] refactor: Pass kwargs into query class #268 - This allows the `COUNT` keyword to be specified during a search API endpoint --- datagateway_api/src/search_api/query.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/datagateway_api/src/search_api/query.py b/datagateway_api/src/search_api/query.py index ecf4d95a..fba70709 100644 --- a/datagateway_api/src/search_api/query.py +++ b/datagateway_api/src/search_api/query.py @@ -4,13 +4,15 @@ class SearchAPIQuery: - def __init__(self, panosc_entity_name): + def __init__(self, panosc_entity_name, **kwargs): self.panosc_entity_name = panosc_entity_name self.icat_entity_name = mappings.mappings[panosc_entity_name][ "base_icat_entity" ] - self.icat_query = ICATQuery(SessionHandler.client, self.icat_entity_name) + self.icat_query = ICATQuery( + SessionHandler.client, self.icat_entity_name, **kwargs, + ) def __repr__(self): return ( From b0f4b1358abb309656eb7b206cc025b516254094 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 13 Jan 2022 17:11:22 +0000 Subject: [PATCH 02/22] refactor: change method signatures of search API endpoint helpers #268 - The endpoint name won't actually be used in the endpoint code, only the entity name will be - Also changed `get_with_id()` to `get_with_pid()` as all use cases will be with a persistent identifier --- datagateway_api/src/api_start_utils.py | 18 +++++------------ .../src/resources/search_api_endpoints.py | 20 +++++++++---------- datagateway_api/src/search_api/helpers.py | 10 +++++----- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/datagateway_api/src/api_start_utils.py b/datagateway_api/src/api_start_utils.py index 5972fc15..dd8b0437 100644 --- a/datagateway_api/src/api_start_utils.py +++ b/datagateway_api/src/api_start_utils.py @@ -233,9 +233,7 @@ def create_api_endpoints(flask_app, api, spec): } for endpoint_name, entity_name in search_api_entity_endpoints.items(): - get_search_endpoint_resource = get_search_endpoint( - endpoint_name, entity_name, - ) + get_search_endpoint_resource = get_search_endpoint(entity_name) api.add_resource( get_search_endpoint_resource, f"{search_api_extension}/{endpoint_name}", @@ -243,9 +241,7 @@ def create_api_endpoints(flask_app, api, spec): ) spec.path(resource=get_search_endpoint_resource, api=api) - get_single_endpoint_resource = get_single_endpoint( - endpoint_name, entity_name, - ) + get_single_endpoint_resource = get_single_endpoint(entity_name) api.add_resource( get_single_endpoint_resource, f"{search_api_extension}/{endpoint_name}/", @@ -253,9 +249,7 @@ def create_api_endpoints(flask_app, api, spec): ) spec.path(resource=get_single_endpoint_resource, api=api) - get_number_count_endpoint_resource = get_number_count_endpoint( - endpoint_name, entity_name, - ) + get_number_count_endpoint_resource = get_number_count_endpoint(entity_name) api.add_resource( get_number_count_endpoint_resource, f"{search_api_extension}/{endpoint_name}/count", @@ -263,9 +257,7 @@ def create_api_endpoints(flask_app, api, spec): ) spec.path(resource=get_number_count_endpoint_resource, api=api) - get_files_endpoint_resource = get_files_endpoint( - search_api_entity_endpoints["datasets"], "datasets", - ) + get_files_endpoint_resource = get_files_endpoint("File") api.add_resource( get_files_endpoint_resource, f"{search_api_extension}/datasets//files", @@ -274,7 +266,7 @@ def create_api_endpoints(flask_app, api, spec): spec.path(resource=get_files_endpoint_resource, api=api) get_number_count_files_endpoint_resource = get_number_count_files_endpoint( - search_api_entity_endpoints["datasets"], "datasets", + "File", ) api.add_resource( get_number_count_files_endpoint_resource, diff --git a/datagateway_api/src/resources/search_api_endpoints.py b/datagateway_api/src/resources/search_api_endpoints.py index 180ab7bd..586fb2f3 100644 --- a/datagateway_api/src/resources/search_api_endpoints.py +++ b/datagateway_api/src/resources/search_api_endpoints.py @@ -14,7 +14,7 @@ log = logging.getLogger() -def get_search_endpoint(endpoint_name, entity_name): +def get_search_endpoint(entity_name): """ TODO - Add docstring """ @@ -23,7 +23,7 @@ class Endpoint(Resource): def get(self): filters = get_filters_from_query_string("search_api", entity_name) log.debug("Filters: %s", filters) - return get_search(endpoint_name, entity_name, filters), 200 + return get_search(entity_name, filters), 200 # TODO - Add `get.__doc__` @@ -31,7 +31,7 @@ def get(self): return Endpoint -def get_single_endpoint(endpoint_name, entity_name): +def get_single_endpoint(entity_name): """ TODO - Add docstring """ @@ -40,7 +40,7 @@ class EndpointWithID(Resource): def get(self, pid): filters = get_filters_from_query_string("search_api", entity_name) log.debug("Filters: %s", filters) - return get_with_id(entity_name, pid), 200 + return get_with_pid(entity_name, pid, filters), 200 # TODO - Add `get.__doc__` @@ -48,7 +48,7 @@ def get(self, pid): return EndpointWithID -def get_number_count_endpoint(endpoint_name, entity_name): +def get_number_count_endpoint(entity_name): """ TODO - Add docstring """ @@ -58,7 +58,7 @@ def get(self): # Only WHERE included on count endpoints filters = get_filters_from_query_string("search_api", entity_name) log.debug("Filters: %s", filters) - return get_count(entity_name), 200 + return get_count(entity_name, filters), 200 # TODO - Add `get.__doc__` @@ -66,7 +66,7 @@ def get(self): return CountEndpoint -def get_files_endpoint(endpoint_name, entity_name): +def get_files_endpoint(entity_name): """ TODO - Add docstring """ @@ -75,7 +75,7 @@ class FilesEndpoint(Resource): def get(self, pid): filters = get_filters_from_query_string("search_api", entity_name) log.debug("Filters: %s", filters) - return get_files(entity_name), 200 + return get_files(entity_name, pid, filters), 200 # TODO - Add `get.__doc__` @@ -83,7 +83,7 @@ def get(self, pid): return FilesEndpoint -def get_number_count_files_endpoint(endpoint_name, entity_name): +def get_number_count_files_endpoint(entity_name): """ TODO - Add docstring """ @@ -93,7 +93,7 @@ def get(self, pid): # Only WHERE included on count endpoints filters = get_filters_from_query_string("search_api", entity_name) log.debug("Filters: %s", filters) - return get_files_count(entity_name, pid) + return get_files_count(entity_name, filters, pid) # TODO - Add `get.__doc__` diff --git a/datagateway_api/src/search_api/helpers.py b/datagateway_api/src/search_api/helpers.py index cf67833d..ffc67bb8 100644 --- a/datagateway_api/src/search_api/helpers.py +++ b/datagateway_api/src/search_api/helpers.py @@ -12,7 +12,7 @@ @client_manager -def get_search(endpoint_name, entity_name, filters): +def get_search(entity_name, filters): log.debug("Entity Name: %s, Filters: %s", entity_name, filters) query = SearchAPIQuery(entity_name) @@ -28,20 +28,20 @@ def get_search(endpoint_name, entity_name, filters): @client_manager -def get_with_id(endpoint_name, entity_name, id_, filters): +def get_with_pid(entity_name, pid, filters): pass @client_manager -def get_count(endpoint_name, entity_name, filters): +def get_count(entity_name, filters): pass @client_manager -def get_files(endpoint_name, entity_name, filters): +def get_files(entity_name, pid, filters): pass @client_manager -def get_files_count(endpoint_name, entity_name, id_, filters): +def get_files_count(entity_name, filters, pid): pass From 71a0852707cd34773f04c2c3baecbe1e2df9cf7a Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 13 Jan 2022 17:13:28 +0000 Subject: [PATCH 03/22] refactor: change expected data type of pid #268 - This matches the PaNOSC data model --- datagateway_api/src/api_start_utils.py | 6 +++--- datagateway_api/src/resources/search_api_endpoints.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datagateway_api/src/api_start_utils.py b/datagateway_api/src/api_start_utils.py index dd8b0437..a7a00236 100644 --- a/datagateway_api/src/api_start_utils.py +++ b/datagateway_api/src/api_start_utils.py @@ -244,7 +244,7 @@ def create_api_endpoints(flask_app, api, spec): get_single_endpoint_resource = get_single_endpoint(entity_name) api.add_resource( get_single_endpoint_resource, - f"{search_api_extension}/{endpoint_name}/", + f"{search_api_extension}/{endpoint_name}/", endpoint=f"search_api_get_single_{endpoint_name}", ) spec.path(resource=get_single_endpoint_resource, api=api) @@ -260,7 +260,7 @@ def create_api_endpoints(flask_app, api, spec): get_files_endpoint_resource = get_files_endpoint("File") api.add_resource( get_files_endpoint_resource, - f"{search_api_extension}/datasets//files", + f"{search_api_extension}/datasets//files", endpoint="search_api_get_dataset_files", ) spec.path(resource=get_files_endpoint_resource, api=api) @@ -270,7 +270,7 @@ def create_api_endpoints(flask_app, api, spec): ) api.add_resource( get_number_count_files_endpoint_resource, - f"{search_api_extension}/datasets//files/count", + f"{search_api_extension}/datasets//files/count", endpoint="search_api_count_dataset_files", ) spec.path(resource=get_number_count_files_endpoint_resource, api=api) diff --git a/datagateway_api/src/resources/search_api_endpoints.py b/datagateway_api/src/resources/search_api_endpoints.py index 586fb2f3..5ff05276 100644 --- a/datagateway_api/src/resources/search_api_endpoints.py +++ b/datagateway_api/src/resources/search_api_endpoints.py @@ -8,7 +8,7 @@ get_files, get_files_count, get_search, - get_with_id, + get_with_pid, ) log = logging.getLogger() From a5aee61cc55e70931163371f3c1abecb31b1fb3a Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 13 Jan 2022 17:15:02 +0000 Subject: [PATCH 04/22] docs: add docstrings for Flask resource classes #268 - Also updated the existing ones for DataGateway API --- .../src/resources/entities/entity_endpoint.py | 24 +++++------ .../src/resources/search_api_endpoints.py | 41 ++++++++++++++++--- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/datagateway_api/src/resources/entities/entity_endpoint.py b/datagateway_api/src/resources/entities/entity_endpoint.py index 45bb2788..bdbc23db 100644 --- a/datagateway_api/src/resources/entities/entity_endpoint.py +++ b/datagateway_api/src/resources/entities/entity_endpoint.py @@ -9,9 +9,9 @@ def get_endpoint(name, entity_type, backend, **kwargs): """ - Given an entity name generate a flask_restful Resource class. - In main.py these generated classes are registered with the api e.g - api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles") + Given an entity name, generate a flask_restful `Resource` class. In + `create_api_endpoints()`, these generated classes are registered with the API e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles") :param name: The name of the entity :type name: :class:`str` @@ -168,9 +168,9 @@ def patch(self): def get_id_endpoint(name, entity_type, backend, **kwargs): """ - Given an entity name generate a flask_restful Resource class. - In main.py these generated classes are registered with the api e.g - api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/") + Given an entity name, generate a flask_restful `Resource` class. In + `create_api_endpoints()`, these generated classes are registered with the API e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/")` :param name: The name of the entity :type name: :class:`str` @@ -306,9 +306,9 @@ def patch(self, id_): def get_count_endpoint(name, entity_type, backend, **kwargs): """ - Given an entity name generate a flask_restful Resource class. - In main.py these generated classes are registered with the api e.g - api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/count") + Given an entity name, generate a flask_restful `Resource` class. In + `create_api_endpoints()`, these generated classes are registered with the API e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/count")` :param name: The name of the entity :type name: :class:`str` @@ -363,9 +363,9 @@ def get(self): def get_find_one_endpoint(name, entity_type, backend, **kwargs): """ - Given an entity name generate a flask_restful Resource class. - In main.py these generated classes are registered with the api e.g - api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/findone") + Given an entity name, generate a flask_restful `Resource` class. In + `create_api_endpoints()`, these generated classes are registered with the API e.g. + `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/findone")` :param name: The name of the entity :type name: :class:`str` diff --git a/datagateway_api/src/resources/search_api_endpoints.py b/datagateway_api/src/resources/search_api_endpoints.py index 5ff05276..2e3e8e0e 100644 --- a/datagateway_api/src/resources/search_api_endpoints.py +++ b/datagateway_api/src/resources/search_api_endpoints.py @@ -16,7 +16,13 @@ def get_search_endpoint(entity_name): """ - TODO - Add docstring + Given an entity name, generate a flask_restful `Resource` class. In + `create_api_endpoints()`, these generated classes are registered with the API e.g. + `api.add_resource(get_search_endpoint("Dataset"), "/datasets")` + + :param entity_name: Name of the entity + :type entity_name: :class:`str` + :return: Generated endpoint class """ class Endpoint(Resource): @@ -33,7 +39,13 @@ def get(self): def get_single_endpoint(entity_name): """ - TODO - Add docstring + Given an entity name, generate a flask_restful `Resource` class. In + `create_api_endpoints()`, these generated classes are registered with the API e.g. + `api.add_resource(get_single_endpoint("Dataset"), "/datasets/")` + + :param entity_name: Name of the entity + :type entity_name: :class:`str` + :return: Generated endpoint class """ class EndpointWithID(Resource): @@ -50,7 +62,13 @@ def get(self, pid): def get_number_count_endpoint(entity_name): """ - TODO - Add docstring + Given an entity name, generate a flask_restful `Resource` class. In + `create_api_endpoints()`, these generated classes are registered with the API e.g. + `api.add_resource(get_number_count_endpoint("Dataset"), "/datasets/count")` + + :param entity_name: Name of the entity + :type entity_name: :class:`str` + :return: Generated endpoint class """ class CountEndpoint(Resource): @@ -68,7 +86,13 @@ def get(self): def get_files_endpoint(entity_name): """ - TODO - Add docstring + Given an entity name, generate a flask_restful `Resource` class. In + `create_api_endpoints()`, these generated classes are registered with the API e.g. + `api.add_resource(get_files_endpoint("Dataset"), "/datasets//files")` + + :param entity_name: Name of the entity + :type entity_name: :class:`str` + :return: Generated endpoint class """ class FilesEndpoint(Resource): @@ -85,7 +109,14 @@ def get(self, pid): def get_number_count_files_endpoint(entity_name): """ - TODO - Add docstring + Given an entity name, generate a flask_restful `Resource` class. In + `create_api_endpoints()`, these generated classes are registered with the API e.g. + `api.add_resource(get_number_count_files_endpoint("Dataset"), + "/datasets/files/count")` + + :param entity_name: Name of the entity + :type entity_name: :class:`str` + :return: Generated endpoint class """ class CountFilesEndpoint(Resource): From dcc332e352ded8af25dce7dae635bd62417d2c13 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 13 Jan 2022 17:16:28 +0000 Subject: [PATCH 05/22] feat: implement search API endpoints #266, #267, #268 --- datagateway_api/src/search_api/helpers.py | 139 ++++++++++++++++++++-- 1 file changed, 131 insertions(+), 8 deletions(-) diff --git a/datagateway_api/src/search_api/helpers.py b/datagateway_api/src/search_api/helpers.py index ffc67bb8..50148372 100644 --- a/datagateway_api/src/search_api/helpers.py +++ b/datagateway_api/src/search_api/helpers.py @@ -1,6 +1,10 @@ +import json import logging +from datagateway_api.src.common.exceptions import MissingRecordError from datagateway_api.src.datagateway_api.filter_order_handler import FilterOrderHandler +from datagateway_api.src.search_api.filters import SearchAPIWhereFilter +import datagateway_api.src.search_api.models as models from datagateway_api.src.search_api.query import SearchAPIQuery from datagateway_api.src.search_api.session_handler import ( client_manager, @@ -13,35 +17,154 @@ @client_manager def get_search(entity_name, filters): + """ + Search for data on the given entity, using filters from the request to restrict the + query + + :param entity_name: Name of the entity requested to query against + :type entity_name: :class:`str` + :param filters: The list of Search API filters to be applied to the request/query + :type filters: List of specific implementation :class:`QueryFilter` + :return: List of records (in JSON serialisable format) of the given entity for the + query constructed from that and the request's filters + """ + + log.info("Searching for %s using request's filters", entity_name) log.debug("Entity Name: %s, Filters: %s", entity_name, filters) query = SearchAPIQuery(entity_name) filter_handler = FilterOrderHandler() filter_handler.add_filters(filters) + filter_handler.merge_python_icat_limit_skip_filters() filter_handler.apply_filters(query) - log.debug("Python ICAT Query: %s", query.icat_query.query) - # TODO - `getApiVersion()` used as a placeholder for testing client handling - # Replace with endpoint functionality when implementing the endpoints - return SessionHandler.client.getApiVersion() + log.debug("JPQL Query to be sent/executed in ICAT: %s", query.icat_query.query) + icat_query_data = query.icat_query.execute_query(SessionHandler.client, True) + + panosc_data = [] + for icat_data in icat_query_data: + panosc_model = getattr(models, entity_name) + panosc_record = panosc_model.from_icat(icat_data).json(by_alias=True) + panosc_data.append(json.loads(panosc_record)) + + return panosc_data @client_manager def get_with_pid(entity_name, pid, filters): - pass + """ + Get a particular record of data from the specified entity + + These will only be called with entity names of Dataset, Document and Instrument. + Each of these entities have a PID attribute, so we can assume the identifier will be + persistent (or `pid`) rather than an ordinary identifier (`id`) + + :param entity_name: Name of the entity requested to query against + :type entity_name: :class:`str` + :param pid: Persistent identifier of the data to find + :type pid: :class:`str` + :param filters: The list of Search API filters to be applied to the request/query + :type filters: List of specific implementation :class:`QueryFilter` + :return: The (in JSON serialisable format) record of the specified PID + :raises MissingRecordError: If no results can be found for the query + """ + + log.info("Getting %s from ID %s", entity_name, pid) + log.debug("Entity Name: %s, Filters: %s", entity_name, filters) + + filters.append(SearchAPIWhereFilter("pid", "eq", pid)) + + query = SearchAPIQuery(entity_name) + + filter_handler = FilterOrderHandler() + filter_handler.add_filters(filters) + filter_handler.merge_python_icat_limit_skip_filters() + filter_handler.apply_filters(query) + + log.debug("JPQL Query to be sent/executed in ICAT: %s", query.icat_query.query) + icat_query_data = query.icat_query.execute_query(SessionHandler.client, True) + + if not icat_query_data: + raise MissingRecordError("No result found") + else: + return icat_query_data[0] @client_manager def get_count(entity_name, filters): - pass + """ + Get the number of results of a given entity, with filters provided in the request to + restrict the search + + :param entity_name: Name of the entity requested to query against + :type entity_name: :class:`str` + :param filters: The list of Search API filters to be applied to the request/query + :type filters: List of specific implementation :class:`QueryFilter` + :return: Dict containing the number of records returned from the query + """ + + log.info("Getting number of results for %s, using request's filters", entity_name) + log.debug("Entity Name: %s, Filters: %s", entity_name, filters) + + query = SearchAPIQuery(entity_name, aggregate="COUNT") + + filter_handler = FilterOrderHandler() + filter_handler.add_filters(filters) + filter_handler.merge_python_icat_limit_skip_filters() + filter_handler.apply_filters(query) + + log.debug("Python ICAT Query: %s", query.icat_query.query) + + log.debug("JPQL Query to be sent/executed in ICAT: %s", query.icat_query.query) + icat_query_data = query.icat_query.execute_query(SessionHandler.client, True) + + return {"count": icat_query_data[0]} @client_manager def get_files(entity_name, pid, filters): - pass + """ + Using the PID of a dataset, find all of its associated files and return them + + :param entity_name: Name of the entity requested to query against + :type entity_name: :class:`str` + :param pid: Persistent identifier of the dataset + :type pid: :class:`str` + :param filters: The list of Search API filters to be applied to the request/query + :type filters: List of specific implementation :class:`QueryFilter` + :return: List of file records for the dataset given by PID + """ + + log.info("Getting files of dataset (PID: %s), using request's filters", pid) + log.debug( + "Entity Name: %s, Filters: %s", entity_name, filters, + ) + + filters.append(SearchAPIWhereFilter("dataset.pid", pid, "eq")) + return get_search(entity_name, filters) @client_manager def get_files_count(entity_name, filters, pid): - pass + """ + Using the PID of a dataset, find the number of associated files + + :param entity_name: Name of the entity requested to query against + :type entity_name: :class:`str` + :param pid: Persistent identifier of the data to find + :type pid: :class:`str` + :param filters: The list of Search API filters to be applied to the request/query + :type filters: List of specific implementation :class:`QueryFilter` + :return: Dict containing the number of files for the dataset given by PID + """ + + log.info( + "Getting number of files for dataset (PID: %s), using request's filters", pid, + ) + log.debug( + "Entity Name: %s, Filters: %s", entity_name, filters, + ) + + filters.append(SearchAPIWhereFilter("dataset.pid", pid, "eq")) + return get_count(entity_name, filters) From 1e38eaee9f201a45742cf868ad2fa28f4adee065 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 14 Jan 2022 15:16:12 +0000 Subject: [PATCH 06/22] fix: correct order of arguments for where filter #266 --- datagateway_api/src/search_api/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datagateway_api/src/search_api/helpers.py b/datagateway_api/src/search_api/helpers.py index 50148372..0a5e4e85 100644 --- a/datagateway_api/src/search_api/helpers.py +++ b/datagateway_api/src/search_api/helpers.py @@ -73,7 +73,7 @@ def get_with_pid(entity_name, pid, filters): log.info("Getting %s from ID %s", entity_name, pid) log.debug("Entity Name: %s, Filters: %s", entity_name, filters) - filters.append(SearchAPIWhereFilter("pid", "eq", pid)) + filters.append(SearchAPIWhereFilter("pid", pid, "eq")) query = SearchAPIQuery(entity_name) From a17fb9a0f0b3a2deb91c3c244d3749d994d64cbd Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 14 Jan 2022 16:23:13 +0000 Subject: [PATCH 07/22] test: add basic valid tests for search API endpoints #266, #267, #268 --- test/search_api/conftest.py | 19 +++- .../endpoints/test_count_dataset_files.py | 34 +++++++ .../endpoints/test_count_endpoint.py | 40 ++++++++ .../endpoints/test_get_dataset_files.py | 49 +++++++++ .../endpoints/test_get_entity_by_pid.py | 99 +++++++++++++++++++ .../endpoints/test_search_endpoint.py | 71 +++++++++++++ 6 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 test/search_api/endpoints/test_count_dataset_files.py create mode 100644 test/search_api/endpoints/test_count_endpoint.py create mode 100644 test/search_api/endpoints/test_get_dataset_files.py create mode 100644 test/search_api/endpoints/test_get_entity_by_pid.py create mode 100644 test/search_api/endpoints/test_search_endpoint.py diff --git a/test/search_api/conftest.py b/test/search_api/conftest.py index e5bfb508..09cad13f 100644 --- a/test/search_api/conftest.py +++ b/test/search_api/conftest.py @@ -1,8 +1,14 @@ import json -from unittest.mock import mock_open, patch +from unittest.mock import MagicMock, mock_open, patch +from flask import Flask import pytest +from datagateway_api.src.api_start_utils import ( + create_api_endpoints, + create_app_infrastructure, +) +from datagateway_api.src.common.config import Config from datagateway_api.src.search_api.panosc_mappings import PaNOSCMappings from datagateway_api.src.search_api.query import SearchAPIQuery @@ -132,3 +138,14 @@ def test_panosc_mappings(test_search_api_mappings_data): "builtins.open", mock_open(read_data=json.dumps(test_search_api_mappings_data)), ): return PaNOSCMappings("test/path") + + +@pytest.fixture(scope="package") +def flask_test_app_search_api(flask_test_app): + search_api_app = Flask(__name__) + search_api_app.config["TESTING"] = True + + api, spec = create_app_infrastructure(search_api_app) + create_api_endpoints(search_api_app, api, spec) + + yield search_api_app.test_client() diff --git a/test/search_api/endpoints/test_count_dataset_files.py b/test/search_api/endpoints/test_count_dataset_files.py new file mode 100644 index 00000000..fc61c8ee --- /dev/null +++ b/test/search_api/endpoints/test_count_dataset_files.py @@ -0,0 +1,34 @@ +import pytest + +from datagateway_api.src.common.config import Config + + +class TestSearchAPICountDatasetFilesEndpoint: + @pytest.mark.parametrize( + "pid, request_filter, expected_json", + [ + pytest.param( + "0-8401-1070-7", + "{}", + {"count": 56}, + id="Basic /datasets/{pid}/files/count request", + ), + ], + ) + def test_valid_count_dataset_files_endpoint( + self, flask_test_app_search_api, pid, request_filter, expected_json, + ): + test_response = flask_test_app_search_api.get( + f"{Config.config.search_api.extension}/datasets/{pid}/files/count" + f"?filter={request_filter}", + ) + + print(test_response) + print(test_response.json) + + assert test_response.status_code == 200 + assert test_response.json == expected_json + + def test_invalid_count_dataset_files_endpoint(self): + # TODO - test for bad filter (where only) + pass diff --git a/test/search_api/endpoints/test_count_endpoint.py b/test/search_api/endpoints/test_count_endpoint.py new file mode 100644 index 00000000..f5782498 --- /dev/null +++ b/test/search_api/endpoints/test_count_endpoint.py @@ -0,0 +1,40 @@ +import pytest + +from datagateway_api.src.common.config import Config + + +class TestSearchAPICountEndpoint: + @pytest.mark.parametrize( + "endpoint_name, request_filter, expected_json", + [ + pytest.param( + "datasets", "{}", {"count": 479}, id="Basic /datasets/count request", + ), + pytest.param( + "documents", "{}", {"count": 239}, id="Basic /documents/count request", + ), + pytest.param( + "instruments", + "{}", + {"count": 14}, + id="Basic /instruments/count request", + ), + ], + ) + def test_valid_count_endpoint( + self, flask_test_app_search_api, endpoint_name, request_filter, expected_json, + ): + test_response = flask_test_app_search_api.get( + f"{Config.config.search_api.extension}/{endpoint_name}/count?filter=" + f"{request_filter}", + ) + + print(test_response) + print(test_response.json) + + assert test_response.status_code == 200 + assert test_response.json == expected_json + + def test_invalid_count_endpoint(self): + # TODO - test for bad filter (where only) + pass diff --git a/test/search_api/endpoints/test_get_dataset_files.py b/test/search_api/endpoints/test_get_dataset_files.py new file mode 100644 index 00000000..353e56da --- /dev/null +++ b/test/search_api/endpoints/test_get_dataset_files.py @@ -0,0 +1,49 @@ +import pytest + +from datagateway_api.src.common.config import Config + + +class TestSearchAPIGetDatasetFilesEndpoint: + @pytest.mark.parametrize( + "pid, request_filter, expected_json", + [ + pytest.param( + "0-8401-1070-7", + '{"limit": 2}', + [ + { + "id": 1, + "name": "Datafile 1", + "path": "/hit/consumer/red.jpg", + "size": 199643799, + "dataset": None, + }, + { + "id": 10060, + "name": "Datafile 10060", + "path": "/the/current/next.jpg", + "size": 124327237, + "dataset": None, + }, + ], + id="Basic /datasets/{pid}/files request", + ), + ], + ) + def test_valid_get_dataset_files_endpoint( + self, flask_test_app_search_api, pid, request_filter, expected_json, + ): + test_response = flask_test_app_search_api.get( + f"{Config.config.search_api.extension}/datasets/{pid}/files" + f"?filter={request_filter}", + ) + + print(test_response) + print(test_response.json) + + assert test_response.status_code == 200 + assert test_response.json == expected_json + + def test_invalid_get_dataset_files_endpoint(self): + # TODO - test bad filter and bad pid + pass diff --git a/test/search_api/endpoints/test_get_entity_by_pid.py b/test/search_api/endpoints/test_get_entity_by_pid.py new file mode 100644 index 00000000..d41039b2 --- /dev/null +++ b/test/search_api/endpoints/test_get_entity_by_pid.py @@ -0,0 +1,99 @@ +import pytest + +from datagateway_api.src.common.config import Config + + +class TestSearchAPIGetByPIDEndpoint: + @pytest.mark.parametrize( + "endpoint_name, pid, request_filter, expected_json", + [ + pytest.param( + "datasets", + "0-8401-1070-7", + "{}", + { + "description": "Beat professional blue clear style have. Light" + " final summer. Or hour color maybe word side much team.\nMessage" + " weight official learn especially nature. Himself tax west.", + "modTime": "2006-08-24 01:28:06+00:00", + "modId": "user", + "startDate": "2000-10-13 00:00:00+00:00", + "endDate": "2000-02-10 00:00:00+00:00", + "createId": "user", + "complete": True, + "id": 2, + "name": "DATASET 2", + "doi": "0-8401-1070-7", + "createTime": "2013-04-01 10:56:52+00:00", + "location": "/subject/break.jpeg", + }, + id="Basic /datasets/{pid} request", + ), + pytest.param( + "documents", + "0-449-78690-0", + "{}", + { + "visitId": "42", + "modId": "user", + "name": "INVESTIGATION 1", + "createId": "user", + "createTime": "2002-11-27 06:20:36+00:00", + "doi": "0-449-78690-0", + "id": 1, + "summary": "Season identify professor happen third. Beat" + " professional blue clear style have. Light final summer.", + "endDate": "2000-07-09 00:00:00+00:00", + "modTime": "2005-04-30 19:41:49+00:00", + "releaseDate": "2000-07-05 00:00:00+00:00", + "startDate": "2000-04-03 00:00:00+00:00", + "title": "Including spend increase ability music skill former." + " Agreement director concern once technology sometimes someone" + " staff.\nSuccess pull bar. Laugh senior example.", + }, + id="Basic /documents/{pid} request", + ), + pytest.param( + "instruments", + "2", + "{}", + { + "description": "Former outside source play nearly Congress before" + " necessary. Allow want audience test laugh. Economic body person" + " general attorney. Effort weight prevent possible.", + "modId": "user", + "createTime": "2019-02-19 05:57:03+00:00", + "pid": None, + "createId": "user", + "type": "2", + "name": "INSTRUMENT 2", + "modTime": "2019-01-29 23:33:20+00:00", + "id": 2, + "fullName": "With piece reason late model. House office fly." + " International scene call deep answer audience baby true.\n" + "Indicate education across these. Opportunity design too.", + "url": "https://moore.org/", + }, + id="Basic /instruments/{pid} request", + ), + ], + ) + def test_valid_get_by_pid_endpoint( + self, + flask_test_app_search_api, + endpoint_name, + pid, + request_filter, + expected_json, + ): + test_response = flask_test_app_search_api.get( + f"{Config.config.search_api.extension}/{endpoint_name}/{pid}?filter=" + f"{request_filter}", + ) + + assert test_response.status_code == 200 + assert test_response.json == expected_json + + def test_invalid_get_by_pid_endpoint(self): + # TODO - test for bad filter and bad/unknown PID + pass diff --git a/test/search_api/endpoints/test_search_endpoint.py b/test/search_api/endpoints/test_search_endpoint.py new file mode 100644 index 00000000..8e19d9c2 --- /dev/null +++ b/test/search_api/endpoints/test_search_endpoint.py @@ -0,0 +1,71 @@ +import pytest + +from datagateway_api.src.common.config import Config + + +class TestSearchAPISearchEndpoint: + @pytest.mark.parametrize( + "endpoint_name, request_filter, expected_json", + [ + pytest.param( + "datasets", + '{"limit": 2}', + [ + { + "pid": "0-449-78690-0", + "title": "DATASET 1", + "creationDate": "2002-11-27T06:20:36+00:00", + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], + }, + { + "pid": "0-8401-1070-7", + "title": "DATASET 2", + "creationDate": "2013-04-01T10:56:52+00:00", + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], + }, + ], + id="Basic /datasets request", + ), + pytest.param( + "documents", '{"limit": 2}', [{}, {}], id="Basic /documents request", + ), + # TODO - test data will need changing once facility has been fixed + pytest.param( + "instruments", + '{"limit": 2}', + [ + {"pid": 1, "name": "INSTRUMENT 1", "datasets": None}, + {"pid": 2, "name": "INSTRUMENT 2", "datasets": None}, + ], + id="Basic /instruments request", + ), + ], + ) + def test_valid_search_endpoint( + self, flask_test_app_search_api, endpoint_name, request_filter, expected_json, + ): + print(f"RF: {request_filter}") + test_response = flask_test_app_search_api.get( + f"{Config.config.search_api.extension}/{endpoint_name}?filter=" + f"{request_filter}", + ) + print(test_response) + print(test_response.json) + + assert test_response.json == expected_json + + def test_invalid_search_endpoint(self): + # TODO - test for bad filters + pass From 7ba059cfe3b027be7ed003df58f7965addddd589 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 26 Jan 2022 17:10:51 +0000 Subject: [PATCH 08/22] test: fix existing endpoint test data #268 --- .../endpoints/test_get_dataset_files.py | 4 ++-- .../search_api/endpoints/test_search_endpoint.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/test/search_api/endpoints/test_get_dataset_files.py b/test/search_api/endpoints/test_get_dataset_files.py index 353e56da..5efc9732 100644 --- a/test/search_api/endpoints/test_get_dataset_files.py +++ b/test/search_api/endpoints/test_get_dataset_files.py @@ -12,14 +12,14 @@ class TestSearchAPIGetDatasetFilesEndpoint: '{"limit": 2}', [ { - "id": 1, + "id": "1", "name": "Datafile 1", "path": "/hit/consumer/red.jpg", "size": 199643799, "dataset": None, }, { - "id": 10060, + "id": "10060", "name": "Datafile 10060", "path": "/the/current/next.jpg", "size": 124327237, diff --git a/test/search_api/endpoints/test_search_endpoint.py b/test/search_api/endpoints/test_search_endpoint.py index 8e19d9c2..46e5e3ae 100644 --- a/test/search_api/endpoints/test_search_endpoint.py +++ b/test/search_api/endpoints/test_search_endpoint.py @@ -15,6 +15,7 @@ class TestSearchAPISearchEndpoint: "pid": "0-449-78690-0", "title": "DATASET 1", "creationDate": "2002-11-27T06:20:36+00:00", + "isPublic": True, "size": None, "documents": [], "techniques": [], @@ -27,6 +28,7 @@ class TestSearchAPISearchEndpoint: "pid": "0-8401-1070-7", "title": "DATASET 2", "creationDate": "2013-04-01T10:56:52+00:00", + "isPublic": True, "size": None, "documents": [], "techniques": [], @@ -46,8 +48,18 @@ class TestSearchAPISearchEndpoint: "instruments", '{"limit": 2}', [ - {"pid": 1, "name": "INSTRUMENT 1", "datasets": None}, - {"pid": 2, "name": "INSTRUMENT 2", "datasets": None}, + { + "datasets": [], + "facility": "LILS", + "name": "INSTRUMENT 1", + "pid": "1", + }, + { + "datasets": [], + "facility": "LILS", + "name": "INSTRUMENT 2", + "pid": "2", + }, ], id="Basic /instruments request", ), From 68fa3b3763e2745f3c365d9390777fe327ddd6de Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 26 Jan 2022 17:11:34 +0000 Subject: [PATCH 09/22] refactor: add argument for `required_related_fields` on endpoint #268 --- datagateway_api/src/search_api/helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datagateway_api/src/search_api/helpers.py b/datagateway_api/src/search_api/helpers.py index 0b097ca3..e9ab24d1 100644 --- a/datagateway_api/src/search_api/helpers.py +++ b/datagateway_api/src/search_api/helpers.py @@ -67,7 +67,9 @@ def get_search(entity_name, filters): panosc_data = [] for icat_data in icat_query_data: panosc_model = getattr(models, entity_name) - panosc_record = panosc_model.from_icat(icat_data).json(by_alias=True) + panosc_record = panosc_model.from_icat(icat_data, icat_relations).json( + by_alias=True, + ) panosc_data.append(json.loads(panosc_record)) return panosc_data From 29232c6b2c032c61999118b2f69177f3b9bd5d57 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 28 Jan 2022 13:15:49 +0000 Subject: [PATCH 10/22] fix: add logic to deal with `PythonICATIncludeFilter` that could be related for ICAT relations for non-related PaNOSC fields #268 --- .../src/datagateway_api/filter_order_handler.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/datagateway_api/src/datagateway_api/filter_order_handler.py b/datagateway_api/src/datagateway_api/filter_order_handler.py index 7c4615e4..6e54b7d0 100644 --- a/datagateway_api/src/datagateway_api/filter_order_handler.py +++ b/datagateway_api/src/datagateway_api/filter_order_handler.py @@ -1,10 +1,12 @@ import logging from datagateway_api.src.datagateway_api.icat.filters import ( + PythonICATIncludeFilter, PythonICATLimitFilter, PythonICATOrderFilter, PythonICATSkipFilter, ) +from datagateway_api.src.search_api.query import SearchAPIQuery log = logging.getLogger() @@ -42,6 +44,13 @@ def apply_filters(self, query): self.sort_filters() for query_filter in self.filters: + # Using `type()` because we only want the Python ICAT version, don't want + # the code to catch objects that inherit from the class e.g. + # `SearchAPIIncludeFilter` + if type(query_filter) is PythonICATIncludeFilter and isinstance( + query, SearchAPIQuery, + ): + query = query.icat_query.query query_filter.apply_filter(query) def merge_python_icat_limit_skip_filters(self): From bc2aabd0199766a883d25ba099aa5be98c511b2c Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 28 Jan 2022 13:24:57 +0000 Subject: [PATCH 11/22] test: add search API endpoints #268 - Lots of skips have been added for bugs I've found and need to be fixed --- .../endpoints/test_count_dataset_files.py | 62 +++- .../endpoints/test_count_endpoint.py | 101 +++++- .../endpoints/test_get_dataset_files.py | 121 ++++++- .../endpoints/test_get_entity_by_pid.py | 298 +++++++++++++++- .../endpoints/test_search_endpoint.py | 327 +++++++++++++++++- 5 files changed, 879 insertions(+), 30 deletions(-) diff --git a/test/search_api/endpoints/test_count_dataset_files.py b/test/search_api/endpoints/test_count_dataset_files.py index fc61c8ee..32749a1f 100644 --- a/test/search_api/endpoints/test_count_dataset_files.py +++ b/test/search_api/endpoints/test_count_dataset_files.py @@ -12,6 +12,40 @@ class TestSearchAPICountDatasetFilesEndpoint: "{}", {"count": 56}, id="Basic /datasets/{pid}/files/count request", + # Skipped because empty dict for filter doesn't work on where + marks=pytest.mark.skip, + ), + pytest.param( + "0-8401-1070-7", + '{"name": "Datafile 10060"}', + {"count": 1}, + id="Count dataset files with name condition", + ), + pytest.param( + "0-8401-1070-7", + '{"name": {"nlike": "Datafile 10060"}}', + {"count": 55}, + id="Count dataset files with name condition (operator specified)", + ), + pytest.param( + "0-8401-1070-7", + '{"size": {"gt": 50000000}}', + {"count": 40}, + id="Count dataset files with size condition", + ), + pytest.param( + "0-8401-1070-7", + '{"name": "Unknown Datafile"}', + {"count": 0}, + id="Count dataset files with filter to return zero count", + ), + pytest.param( + "unknown pid", + "{}", + {"count": 0}, + id="Non-existent dataset pid", + # Skipped because empty dict for filter doesn't work on where + marks=pytest.mark.skip, ), ], ) @@ -20,15 +54,29 @@ def test_valid_count_dataset_files_endpoint( ): test_response = flask_test_app_search_api.get( f"{Config.config.search_api.extension}/datasets/{pid}/files/count" - f"?filter={request_filter}", + f"?where={request_filter}", ) - print(test_response) - print(test_response.json) - assert test_response.status_code == 200 assert test_response.json == expected_json - def test_invalid_count_dataset_files_endpoint(self): - # TODO - test for bad filter (where only) - pass + @pytest.mark.parametrize( + "pid, request_filter", + [ + pytest.param("0-8401-1070-7", '{"bad filter"}', id="Bad filter"), + pytest.param( + "0-8401-1070-7", + '{"where": {"name": "FILE 4"}}', + id="Where filter inside where query param", + ), + ], + ) + def test_invalid_count_dataset_files_endpoint( + self, flask_test_app_search_api, pid, request_filter, + ): + test_response = flask_test_app_search_api.get( + f"{Config.config.search_api.extension}/datasets/{pid}/files/count" + f"?where={request_filter}", + ) + + assert test_response.status_code == 400 diff --git a/test/search_api/endpoints/test_count_endpoint.py b/test/search_api/endpoints/test_count_endpoint.py index f5782498..64f0c285 100644 --- a/test/search_api/endpoints/test_count_endpoint.py +++ b/test/search_api/endpoints/test_count_endpoint.py @@ -8,16 +8,86 @@ class TestSearchAPICountEndpoint: "endpoint_name, request_filter, expected_json", [ pytest.param( - "datasets", "{}", {"count": 479}, id="Basic /datasets/count request", + "datasets", + "{}", + {"count": 479}, + id="Basic /datasets/count request", + # Skipped because empty dict for filter doesn't work on where + marks=pytest.mark.skip, ), pytest.param( - "documents", "{}", {"count": 239}, id="Basic /documents/count request", + "documents", + "{}", + {"count": 239}, + id="Basic /documents/count request", + # Skipped because empty dict for filter doesn't work on where + marks=pytest.mark.skip, ), pytest.param( "instruments", "{}", {"count": 14}, id="Basic /instruments/count request", + # Skipped because empty dict for filter doesn't work on where + marks=pytest.mark.skip, + ), + pytest.param( + "datasets", + '{"title": "DATASET 30"}', + {"count": 1}, + id="Dataset count with basic where", + ), + pytest.param( + "documents", + '{"title": "INVESTIGATION 2"}', + {"count": 1}, + id="Document count with basic where", + ), + pytest.param( + "instruments", + '{"name": "INSTRUMENT 12"}', + {"count": 1}, + id="Instrument count with basic where", + ), + pytest.param( + "datasets", + '{"title": {"like": "DATASET 30"}}', + {"count": 11}, + id="Dataset count with where (operator specified)", + ), + pytest.param( + "documents", + '{"summary": {"ilike": "nature"}}', + {"count": 7}, + id="Document count with where (operator specified)", + ), + pytest.param( + "instruments", + '{"name": {"nilike": "INSTRUMENT 5"}}', + {"count": 13}, + id="Instrument count with where (operator specified)", + ), + pytest.param( + "datasets", + '{"isPublic": true}', + {"count": 479}, + id="Dataset count with isPublic condition", + # Skipped because the where for isPublic doesn't work + marks=pytest.mark.skip, + ), + pytest.param( + "documents", + '{"isPublic": true}', + {"count": 239}, + id="Document count with isPublic condition", + # Skipped because the where for isPublic doesn't work + marks=pytest.mark.skip, + ), + pytest.param( + "instruments", + '{"facility": {"like": "LILS"}}', + {"count": 14}, + id="Instrument count with where using related ICAT mapping", ), ], ) @@ -25,16 +95,29 @@ def test_valid_count_endpoint( self, flask_test_app_search_api, endpoint_name, request_filter, expected_json, ): test_response = flask_test_app_search_api.get( - f"{Config.config.search_api.extension}/{endpoint_name}/count?filter=" + f"{Config.config.search_api.extension}/{endpoint_name}/count?where=" f"{request_filter}", ) - print(test_response) - print(test_response.json) - assert test_response.status_code == 200 assert test_response.json == expected_json - def test_invalid_count_endpoint(self): - # TODO - test for bad filter (where only) - pass + @pytest.mark.parametrize( + "request_filter", + [ + pytest.param('{"bad filter"}', id="Bad filter"), + pytest.param( + '{"where": {"title": "DATASET 4"}}', + id="Where filter inside where query param", + ), + ], + ) + def test_invalid_count_endpoint( + self, flask_test_app_search_api, request_filter, + ): + test_response = flask_test_app_search_api.get( + f"{Config.config.search_api.extension}/datasets/count" + f"?where={request_filter}", + ) + + assert test_response.status_code == 400 diff --git a/test/search_api/endpoints/test_get_dataset_files.py b/test/search_api/endpoints/test_get_dataset_files.py index 5efc9732..58ca1e6e 100644 --- a/test/search_api/endpoints/test_get_dataset_files.py +++ b/test/search_api/endpoints/test_get_dataset_files.py @@ -28,6 +28,94 @@ class TestSearchAPIGetDatasetFilesEndpoint: ], id="Basic /datasets/{pid}/files request", ), + pytest.param( + "0-8401-1070-7", + '{"limit": 1, "skip": 5}', + [ + { + "id": "11976", + "name": "Datafile 11976", + "path": "/ahead/article/oil.jpg", + "size": 34418452, + "dataset": None, + }, + ], + id="Get dataset files with skip", + ), + pytest.param( + "0-8401-1070-7", + '{"limit": 1, "where": {"name": "Datafile 10060"}}', + [ + { + "id": "10060", + "name": "Datafile 10060", + "path": "/the/current/next.jpg", + "size": 124327237, + "dataset": None, + }, + ], + id="Get dataset files with name condition", + ), + pytest.param( + "0-8401-1070-7", + '{"limit": 1, "where": {"name": {"nilike": "Datafile 10060"}}}', + [ + { + "id": "1", + "name": "Datafile 1", + "path": "/hit/consumer/red.jpg", + "size": 199643799, + "dataset": None, + }, + ], + id="Get dataset files with name condition (operator specified)", + ), + pytest.param( + "0-8401-1070-7", + '{"limit": 1, "where": {"size": {"gt": 5000000}}}', + [ + { + "id": "1", + "name": "Datafile 1", + "path": "/hit/consumer/red.jpg", + "size": 199643799, + "dataset": None, + }, + ], + id="Get dataset files with size condition", + ), + pytest.param( + "0-8401-1070-7", + '{"limit": 1, "where": {"size": {"gt": 50000000000}}}', + [], + id="Get dataset files with condition to return empty list", + ), + pytest.param( + "0-8401-1070-7", + '{"limit": 1, "include": [{"relation": "dataset"}]}', + [ + { + "id": "1", + "name": "Datafile 1", + "path": "/hit/consumer/red.jpg", + "size": 199643799, + "dataset": { + "pid": "0-8401-1070-7", + "title": "DATASET 2", + "isPublic": True, + "creationDate": "2013-04-01T10:56:52+00:00", + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], + }, + }, + ], + id="Get dataset files with include filter", + ), ], ) def test_valid_get_dataset_files_endpoint( @@ -44,6 +132,33 @@ def test_valid_get_dataset_files_endpoint( assert test_response.status_code == 200 assert test_response.json == expected_json - def test_invalid_get_dataset_files_endpoint(self): - # TODO - test bad filter and bad pid - pass + @pytest.mark.parametrize( + "pid, request_filter, expected_status_code", + [ + pytest.param("0-8401-1070-7", '{"where": []}', 400, id="Bad where filter"), + pytest.param("0-8401-1070-7", '{"limit": -1}', 400, id="Bad limit filter"), + pytest.param( + "0-8401-1070-7", '{"limit": -100}', 400, id="Bad skip filter", + ), + pytest.param( + "0-8401-1070-7", '{"include": ""}', 400, id="Bad include filter", + ), + pytest.param( + "my 404 test pid", + "{}", + 404, + id="Non-existent dataset pid", + # Skipped because this actually returns 200 + marks=pytest.mark.skip, + ), + ], + ) + def test_invalid_get_dataset_files_endpoint( + self, flask_test_app_search_api, pid, request_filter, expected_status_code, + ): + test_response = flask_test_app_search_api.get( + f"{Config.config.search_api.extension}/datasets/{pid}/files" + f"?filter={request_filter}", + ) + + assert test_response.status_code == expected_status_code diff --git a/test/search_api/endpoints/test_get_entity_by_pid.py b/test/search_api/endpoints/test_get_entity_by_pid.py index d41039b2..bbef11ea 100644 --- a/test/search_api/endpoints/test_get_entity_by_pid.py +++ b/test/search_api/endpoints/test_get_entity_by_pid.py @@ -76,6 +76,278 @@ class TestSearchAPIGetByPIDEndpoint: }, id="Basic /instruments/{pid} request", ), + pytest.param( + "datasets", + "0-8401-1070-7", + '{"include": [{"relation": "documents"}]}', + { + "createId": "user", + "startDate": "2000-10-13 00:00:00+00:00", + "doi": "0-8401-1070-7", + "modTime": "2006-08-24 01:28:06+00:00", + "createTime": "2013-04-01 10:56:52+00:00", + "location": "/subject/break.jpeg", + "endDate": "2000-02-10 00:00:00+00:00", + "complete": True, + "modId": "user", + "documents": [ + { + "createId": "user", + "doi": "0-9729806-3-6", + "startDate": "2000-06-04 00:00:00+00:00", + "modTime": "2016-11-16 19:42:34+00:00", + "createTime": "2004-08-23 02:41:19+00:00", + "endDate": "2000-09-14 00:00:00+00:00", + "modId": "user", + "title": "Show fly image herself yard challenge by. Past" + " site her number. Not weight half far move. Leader" + " everyone skin still.\nProve begin boy those always" + " dream write inside.", + "summary": "Day purpose item create. Visit hope mean admit." + " The tonight adult cut foreign would situation fund.\n" + "Purpose study usually gas think. Machine world doctor" + " rise be college treat.", + "visitId": "4", + "name": "INVESTIGATION 2", + "releaseDate": "2000-02-10 00:00:00+00:00", + "id": 2, + }, + ], + "description": "Beat professional blue clear style have. Light" + " final summer. Or hour color maybe word side much team." + "\nMessage weight official learn especially nature. Himself" + " tax west.", + "name": "DATASET 2", + "id": 2, + }, + id="Get dataset by pid with include filter", + # Skipped because of incorrect document JSON format + marks=pytest.mark.skip, + ), + pytest.param( + "documents", + "0-449-78690-0", + '{"include": [{"relation": "datasets"}]}', + { + "datasets": [ + { + "createId": "user", + "startDate": "2000-05-07 00:00:00+00:00", + "doi": "0-449-78690-0", + "modTime": "2005-04-30 19:41:49+00:00", + "createTime": "2002-11-27 06:20:36+00:00", + "location": "/international/subject.tiff", + "endDate": "2000-07-05 00:00:00+00:00", + "complete": True, + "modId": "user", + "description": "Many last prepare small. Maintain throw" + " hope parent.\nEntire soon option bill fish against power." + "\nRather why rise month shake voice.", + "name": "DATASET 1", + "id": 1, + }, + { + "createId": "user", + "startDate": "2060-01-07 00:00:00+00:00", + "doi": "0-353-84629-5", + "modTime": "2002-09-30 13:03:32+00:00", + "createTime": "2006-11-21 17:10:42+00:00", + "location": "/gun/special.jpeg", + "endDate": "2060-01-17 00:00:00+00:00", + "complete": True, + "modId": "user", + "description": "Single many hope organization reach process" + " I. Health hit total federal describe. Bill firm rate" + " democratic outside.\nLate while our either worry.", + "name": "DATASET 241", + "id": 241, + }, + ], + "visitId": "42", + "modId": "user", + "name": "INVESTIGATION 1", + "createId": "user", + "createTime": "2002-11-27 06:20:36+00:00", + "doi": "0-449-78690-0", + "id": 1, + "summary": "Season identify professor happen third. Beat" + " professional blue clear style have. Light final summer.", + "endDate": "2000-07-09 00:00:00+00:00", + "modTime": "2005-04-30 19:41:49+00:00", + "releaseDate": "2000-07-05 00:00:00+00:00", + "startDate": "2000-04-03 00:00:00+00:00", + "title": "Including spend increase ability music skill former." + " Agreement director concern once technology sometimes someone" + " staff.\nSuccess pull bar. Laugh senior example.", + }, + id="Get document by pid with include filter", + ), + pytest.param( + "instruments", + "2", + '{"include": [{"relation": "datasets"}]}', + { + "description": "Former outside source play nearly Congress before" + " necessary. Allow want audience test laugh. Economic body person" + " general attorney. Effort weight prevent possible.", + "modId": "user", + "createTime": "2019-02-19 05:57:03+00:00", + "pid": None, + "createId": "user", + "type": "2", + "name": "INSTRUMENT 2", + "modTime": "2019-01-29 23:33:20+00:00", + "id": 2, + "fullName": "With piece reason late model. House office fly." + " International scene call deep answer audience baby true.\n" + "Indicate education across these. Opportunity design too.", + "url": "https://moore.org/", + }, + id="Get instrument by pid with include filter", + # Skipped due to ICAT 5 mapping + marks=pytest.mark.skip, + ), + pytest.param( + "datasets", + "0-8401-1070-7", + '{"include": [{"relation": "documents"}, {"relation": "techniques"},' + ' {"relation": "instrument"}, {"relation": "files"},' + ' {"relation": "parameters"}, {"relation": "samples"}]}', + { + "createId": "user", + "startDate": "2000-10-13 00:00:00+00:00", + "doi": "0-8401-1070-7", + "modTime": "2006-08-24 01:28:06+00:00", + "createTime": "2013-04-01 10:56:52+00:00", + "location": "/subject/break.jpeg", + "endDate": "2000-02-10 00:00:00+00:00", + "complete": True, + "modId": "user", + "documents": [ + { + "createId": "user", + "doi": "0-9729806-3-6", + "startDate": "2000-06-04 00:00:00+00:00", + "modTime": "2016-11-16 19:42:34+00:00", + "createTime": "2004-08-23 02:41:19+00:00", + "endDate": "2000-09-14 00:00:00+00:00", + "modId": "user", + "title": "Show fly image herself yard challenge by. Past" + " site her number. Not weight half far move. Leader" + " everyone skin still.\nProve begin boy those always" + " dream write inside.", + "summary": "Day purpose item create. Visit hope mean admit." + " The tonight adult cut foreign would situation fund.\n" + "Purpose study usually gas think. Machine world doctor" + " rise be college treat.", + "visitId": "4", + "name": "INVESTIGATION 2", + "releaseDate": "2000-02-10 00:00:00+00:00", + "id": 2, + }, + ], + "description": "Beat professional blue clear style have. Light" + " final summer. Or hour color maybe word side much team." + "\nMessage weight official learn especially nature. Himself" + " tax west.", + "name": "DATASET 2", + "id": 2, + }, + id="Get dataset by pid including all possible related entities", + # Skipped because of incorrect document JSON format and ICAT 5 mapping + # on techniques and instrument + marks=pytest.mark.skip, + ), + pytest.param( + "documents", + "0-449-78690-0", + '{"include": [{"relation": "datasets"}, {"relation": "members"},' + ' {"relation": "parameters"}]}', + { + "datasets": [ + { + "createId": "user", + "startDate": "2000-05-07 00:00:00+00:00", + "doi": "0-449-78690-0", + "modTime": "2005-04-30 19:41:49+00:00", + "createTime": "2002-11-27 06:20:36+00:00", + "location": "/international/subject.tiff", + "endDate": "2000-07-05 00:00:00+00:00", + "complete": True, + "modId": "user", + "description": "Many last prepare small. Maintain throw" + " hope parent.\nEntire soon option bill fish against power." + "\nRather why rise month shake voice.", + "parameters": [], + "name": "DATASET 1", + "id": 1, + }, + { + "createId": "user", + "startDate": "2060-01-07 00:00:00+00:00", + "doi": "0-353-84629-5", + "modTime": "2002-09-30 13:03:32+00:00", + "createTime": "2006-11-21 17:10:42+00:00", + "location": "/gun/special.jpeg", + "endDate": "2060-01-17 00:00:00+00:00", + "complete": True, + "modId": "user", + "description": "Single many hope organization reach process" + " I. Health hit total federal describe. Bill firm rate" + " democratic outside.\nLate while our either worry.", + "parameters": [], + "name": "DATASET 241", + "id": 241, + }, + ], + "createId": "user", + "doi": "0-449-78690-0", + "startDate": "2000-04-03 00:00:00+00:00", + "modTime": "2005-04-30 19:41:49+00:00", + "createTime": "2002-11-27 06:20:36+00:00", + "members": [ + { + "createId": "user", + "modId": "user", + "modTime": "2005-04-30 19:41:49+00:00", + "role": "CI", + "createTime": "2002-11-27 06:20:36+00:00", + "id": 1, + }, + ], + "endDate": "2000-07-09 00:00:00+00:00", + "modId": "user", + "title": "Including spend increase ability music skill former." + " Agreement director concern once technology sometimes someone" + " staff.\nSuccess pull bar. Laugh senior example.", + "summary": "Season identify professor happen third. Beat" + " professional blue clear style have. Light final summer.", + "visitId": "42", + "parameters": [ + { + "dateTimeValue": "2000-05-07 00:00:00+00:00", + "rangeBottom": 48.0, + "createId": "user", + "numericValue": 127265.0, + "modTime": "2005-04-30 19:41:49+00:00", + "createTime": "2002-11-27 06:20:36+00:00", + "id": 1, + "stringValue": "international1", + "modId": "user", + "rangeTop": 101.0, + "error": 31472.0, + }, + ], + "name": "INVESTIGATION 1", + "releaseDate": "2000-07-05 00:00:00+00:00", + "id": 1, + }, + id="Get document by pid including all possible related entities", + # Skipped because of incorrect members JSON naming + # (investigationUsers key used instead). It's in ICAT fields too, + # not converted into PaNOSC format + marks=pytest.mark.skip, + ), ], ) def test_valid_get_by_pid_endpoint( @@ -94,6 +366,26 @@ def test_valid_get_by_pid_endpoint( assert test_response.status_code == 200 assert test_response.json == expected_json - def test_invalid_get_by_pid_endpoint(self): - # TODO - test for bad filter and bad/unknown PID - pass + @pytest.mark.parametrize( + "pid, request_filter, expected_status_code", + [ + pytest.param("0-8401-1070-7", '{"where": []}', 400, id="Bad where filter"), + pytest.param("0-8401-1070-7", '{"limit": -1}', 400, id="Bad limit filter"), + pytest.param( + "0-8401-1070-7", '{"limit": -100}', 400, id="Bad skip filter", + ), + pytest.param( + "0-8401-1070-7", '{"include": ""}', 400, id="Bad include filter", + ), + pytest.param("my 404 test pid", "{}", 404, id="Non-existent dataset pid"), + ], + ) + def test_invalid_get_by_pid_endpoint( + self, flask_test_app_search_api, pid, request_filter, expected_status_code, + ): + test_response = flask_test_app_search_api.get( + f"{Config.config.search_api.extension}/datasets/{pid}" + f"?filter={request_filter}", + ) + + assert test_response.status_code == expected_status_code diff --git a/test/search_api/endpoints/test_search_endpoint.py b/test/search_api/endpoints/test_search_endpoint.py index 46e5e3ae..38225676 100644 --- a/test/search_api/endpoints/test_search_endpoint.py +++ b/test/search_api/endpoints/test_search_endpoint.py @@ -41,9 +41,108 @@ class TestSearchAPISearchEndpoint: id="Basic /datasets request", ), pytest.param( - "documents", '{"limit": 2}', [{}, {}], id="Basic /documents request", + "documents", + '{"limit": 1}', + [ + { + "pid": "0-449-78690-0", + "isPublic": True, + "type": "INVESTIGATIONTYPE 2", + "title": "INVESTIGATION 1", + "summary": "Season identify professor happen third. Beat" + " professional blue clear style have. Light final summer.", + "doi": "0-449-78690-0", + "startDate": "2000-04-03T00:00:00+00:00", + "endDate": "2000-07-09T00:00:00+00:00", + "releaseDate": "2000-07-05T00:00:00+00:00", + "license": None, + "keywords": [ + "read123", + "boy129", + "out253", + "hour326", + "possible449", + "west566", + "scene948", + "who1253", + "capital1526", + "dream1989", + "front2347", + "inside2465", + "surface2851", + "learn2953", + "hot3053", + "just3159", + "population3261", + "cup3366", + "another3451", + "environmental3632", + "require3858", + "rock3952", + "determine4048", + "space4061", + "big4229", + "why4243", + "public4362", + "election4641", + "measure4996", + "often5014", + "develop5135", + "than5310", + "floor5312", + "check5327", + "cost5487", + "information6130", + "guy6180", + "admit6235", + "market6645", + "law6777", + "close7336", + "billion7597", + "product7964", + "American8041", + "language8246", + "school8277", + "specific8539", + "position8670", + "grow8702", + "time8899", + "weight9086", + "catch9129", + "speak9559", + "strong9621", + "development9757", + "best9786", + "identify10039", + "give10497", + "life10854", + "century11040", + "fire11580", + "leg11744", + "past11935", + "bar12034", + "do12108", + "prove12224", + "body12251", + "data12288", + "at12640", + "star12706", + "customer12795", + "small13058", + "event13141", + "now13193", + "magazine13415", + "policy13601", + "black13996", + "American14654", + ], + "datasets": [], + "members": [], + "parameters": [], + }, + ], + id="Basic /documents request", ), - # TODO - test data will need changing once facility has been fixed pytest.param( "instruments", '{"limit": 2}', @@ -63,21 +162,233 @@ class TestSearchAPISearchEndpoint: ], id="Basic /instruments request", ), + pytest.param( + "datasets", + '{"limit": 1, "skip": 5}', + [ + { + "pid": "0-468-20341-9", + "title": "DATASET 6", + "isPublic": True, + "creationDate": "2008-06-30T08:30:58+00:00", + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], + }, + ], + id="Search datasets with skip filter", + ), + pytest.param( + "instruments", + '{"limit": 2, "where": {"name": "INSTRUMENT 10"}}', + [ + { + "datasets": [], + "facility": "LILS", + "name": "INSTRUMENT 10", + "pid": "10", + }, + ], + id="Search instruments with name condition", + ), + pytest.param( + "datasets", + '{"limit": 1, "where": {"creationDate": {"gt":' + ' "2007-06-30 08:30:58+00:00"}}}', + [ + { + "pid": "0-8401-1070-7", + "title": "DATASET 2", + "isPublic": True, + "creationDate": "2013-04-01T10:56:52+00:00", + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], + }, + ], + id="Search datasets with creation date filter (operator specified)", + ), + pytest.param( + "datasets", + '{"include": [{"relation": "documents"}, {"relation": "techniques"},' + ' {"relation": "instrument"}, {"relation": "files"},' + ' {"relation": "parameters"}, {"relation": "samples"}], "limit": 1}', + [ + { + "createId": "user", + "startDate": "2000-10-13 00:00:00+00:00", + "doi": "0-8401-1070-7", + "modTime": "2006-08-24 01:28:06+00:00", + "createTime": "2013-04-01 10:56:52+00:00", + "location": "/subject/break.jpeg", + "endDate": "2000-02-10 00:00:00+00:00", + "complete": True, + "modId": "user", + "documents": [{}], + "description": "Beat professional blue clear style have. Light" + " final summer. Or hour color maybe word side much team." + "\nMessage weight official learn especially nature. Himself" + " tax west.", + "name": "DATASET 2", + "id": 2, + }, + ], + id="Search datasets including all possible related entities", + # Skipped because of incorrect document JSON format and ICAT 5 mapping + # on techniques and instrument + marks=pytest.mark.skip, + ), + pytest.param( + "documents", + '{"include": [{"relation": "datasets"}, {"relation": "members"},' + ' {"relation": "parameters"}], "limit": 1}', + [ + { + "datasets": [ + { + "createId": "user", + "startDate": "2000-05-07 00:00:00+00:00", + "doi": "0-449-78690-0", + "modTime": "2005-04-30 19:41:49+00:00", + "createTime": "2002-11-27 06:20:36+00:00", + "location": "/international/subject.tiff", + "endDate": "2000-07-05 00:00:00+00:00", + "complete": True, + "modId": "user", + "description": "Many last prepare small. Maintain throw" + " hope parent.\nEntire soon option bill fish against" + " power." + "\nRather why rise month shake voice.", + "parameters": [], + "name": "DATASET 1", + "id": 1, + }, + { + "createId": "user", + "startDate": "2060-01-07 00:00:00+00:00", + "doi": "0-353-84629-5", + "modTime": "2002-09-30 13:03:32+00:00", + "createTime": "2006-11-21 17:10:42+00:00", + "location": "/gun/special.jpeg", + "endDate": "2060-01-17 00:00:00+00:00", + "complete": True, + "modId": "user", + "description": "Single many hope organization reach" + " process I. Health hit total federal describe. Bill" + " firm rate democratic outside.\nLate while our either" + " worry.", + "parameters": [], + "name": "DATASET 241", + "id": 241, + }, + ], + "createId": "user", + "doi": "0-449-78690-0", + "startDate": "2000-04-03 00:00:00+00:00", + "modTime": "2005-04-30 19:41:49+00:00", + "createTime": "2002-11-27 06:20:36+00:00", + "members": [ + { + "createId": "user", + "modId": "user", + "modTime": "2005-04-30 19:41:49+00:00", + "role": "CI", + "createTime": "2002-11-27 06:20:36+00:00", + "id": 1, + }, + ], + "endDate": "2000-07-09 00:00:00+00:00", + "modId": "user", + "title": "Including spend increase ability music skill former." + " Agreement director concern once technology sometimes someone" + " staff.\nSuccess pull bar. Laugh senior example.", + "summary": "Season identify professor happen third. Beat" + " professional blue clear style have. Light final summer.", + "visitId": "42", + "parameters": [ + { + "dateTimeValue": "2000-05-07 00:00:00+00:00", + "rangeBottom": 48.0, + "createId": "user", + "numericValue": 127265.0, + "modTime": "2005-04-30 19:41:49+00:00", + "createTime": "2002-11-27 06:20:36+00:00", + "id": 1, + "stringValue": "international1", + "modId": "user", + "rangeTop": 101.0, + "error": 31472.0, + }, + ], + "name": "INVESTIGATION 1", + "releaseDate": "2000-07-05 00:00:00+00:00", + "id": 1, + }, + ], + id="Search documents including all possible related entities", + # Skipped because of incorrect members JSON naming + # (investigationUsers key used instead). It's in ICAT fields too, + # not converted into PaNOSC format + marks=pytest.mark.skip, + ), + pytest.param( + "instruments", + '{"include": [{"relation": "datasets"}], "limit": 1}', + { + "description": "Former outside source play nearly Congress before" + " necessary. Allow want audience test laugh. Economic body person" + " general attorney. Effort weight prevent possible.", + "modId": "user", + "createTime": "2019-02-19 05:57:03+00:00", + "pid": None, + "createId": "user", + "type": "2", + "name": "INSTRUMENT 2", + "modTime": "2019-01-29 23:33:20+00:00", + "id": 2, + "fullName": "With piece reason late model. House office fly." + " International scene call deep answer audience baby true.\n" + "Indicate education across these. Opportunity design too.", + "url": "https://moore.org/", + }, + id="Search instruments including all possible related entities", + # Skipped due to ICAT 5 mapping + marks=pytest.mark.skip, + ), ], ) def test_valid_search_endpoint( self, flask_test_app_search_api, endpoint_name, request_filter, expected_json, ): - print(f"RF: {request_filter}") test_response = flask_test_app_search_api.get( f"{Config.config.search_api.extension}/{endpoint_name}?filter=" f"{request_filter}", ) - print(test_response) - print(test_response.json) assert test_response.json == expected_json - def test_invalid_search_endpoint(self): - # TODO - test for bad filters - pass + @pytest.mark.parametrize( + "request_filter, expected_status_code", + [ + pytest.param('{"where": []}', 400, id="Bad where filter"), + pytest.param('{"limit": -1}', 400, id="Bad limit filter"), + pytest.param('{"limit": -100}', 400, id="Bad skip filter"), + pytest.param('{"include": ""}', 400, id="Bad include filter"), + ], + ) + def test_invalid_search_endpoint( + self, flask_test_app_search_api, request_filter, expected_status_code, + ): + test_response = flask_test_app_search_api.get( + f"{Config.config.search_api.extension}/instruments?filter={request_filter}", + ) + + assert test_response.status_code == expected_status_code From d14c327cd40a665697b420747e113b44310dcd36 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Fri, 28 Jan 2022 13:25:43 +0000 Subject: [PATCH 12/22] style: general linting fixes #268 --- datagateway_api/src/search_api/filters.py | 1 - datagateway_api/src/search_api/helpers.py | 8 +++++--- test/search_api/conftest.py | 3 +-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datagateway_api/src/search_api/filters.py b/datagateway_api/src/search_api/filters.py index 1ebbf579..cd840204 100644 --- a/datagateway_api/src/search_api/filters.py +++ b/datagateway_api/src/search_api/filters.py @@ -3,7 +3,6 @@ import logging from datagateway_api.src.common.date_handler import DateHandler -from datagateway_api.src.common.exceptions import FilterError from datagateway_api.src.datagateway_api.icat.filters import ( PythonICATIncludeFilter, PythonICATLimitFilter, diff --git a/datagateway_api/src/search_api/helpers.py b/datagateway_api/src/search_api/helpers.py index e9ab24d1..3861fe3e 100644 --- a/datagateway_api/src/search_api/helpers.py +++ b/datagateway_api/src/search_api/helpers.py @@ -3,10 +3,12 @@ from datagateway_api.src.common.exceptions import MissingRecordError from datagateway_api.src.datagateway_api.filter_order_handler import FilterOrderHandler -from datagateway_api.src.search_api.filters import SearchAPIWhereFilter -import datagateway_api.src.search_api.models as models from datagateway_api.src.datagateway_api.icat.filters import PythonICATIncludeFilter -from datagateway_api.src.search_api.filters import SearchAPIIncludeFilter +from datagateway_api.src.search_api.filters import ( + SearchAPIIncludeFilter, + SearchAPIWhereFilter, +) +import datagateway_api.src.search_api.models as models from datagateway_api.src.search_api.panosc_mappings import mappings from datagateway_api.src.search_api.query import SearchAPIQuery from datagateway_api.src.search_api.session_handler import ( diff --git a/test/search_api/conftest.py b/test/search_api/conftest.py index 09cad13f..218124da 100644 --- a/test/search_api/conftest.py +++ b/test/search_api/conftest.py @@ -1,5 +1,5 @@ import json -from unittest.mock import MagicMock, mock_open, patch +from unittest.mock import mock_open, patch from flask import Flask import pytest @@ -8,7 +8,6 @@ create_api_endpoints, create_app_infrastructure, ) -from datagateway_api.src.common.config import Config from datagateway_api.src.search_api.panosc_mappings import PaNOSCMappings from datagateway_api.src.search_api.query import SearchAPIQuery From 12bdde5bdd164a1f7e2c69329c90b084f1f3cb52 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 31 Jan 2022 11:22:34 +0000 Subject: [PATCH 13/22] test: add additional `isPublic` test for endpoints #268 --- test/search_api/endpoints/test_search_endpoint.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/search_api/endpoints/test_search_endpoint.py b/test/search_api/endpoints/test_search_endpoint.py index 38225676..a5258301 100644 --- a/test/search_api/endpoints/test_search_endpoint.py +++ b/test/search_api/endpoints/test_search_endpoint.py @@ -363,6 +363,14 @@ class TestSearchAPISearchEndpoint: # Skipped due to ICAT 5 mapping marks=pytest.mark.skip, ), + pytest.param( + "datasets", + '{"limit": 1, "where": {"isPublic": true}}', + [{}], + id="Search datasets with isPublic condition", + # Skipped because the where for isPublic doesn't work + marks=pytest.mark.skip, + ), ], ) def test_valid_search_endpoint( From 2fed560a82794c68850e1e94f156bc8d96685226 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 31 Jan 2022 17:43:50 +0000 Subject: [PATCH 14/22] test: correct query params on tests #268 --- test/search_api/endpoints/test_get_dataset_files.py | 4 +--- test/search_api/endpoints/test_get_entity_by_pid.py | 4 +--- test/search_api/endpoints/test_search_endpoint.py | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/test/search_api/endpoints/test_get_dataset_files.py b/test/search_api/endpoints/test_get_dataset_files.py index 58ca1e6e..e0d47f6a 100644 --- a/test/search_api/endpoints/test_get_dataset_files.py +++ b/test/search_api/endpoints/test_get_dataset_files.py @@ -137,9 +137,7 @@ def test_valid_get_dataset_files_endpoint( [ pytest.param("0-8401-1070-7", '{"where": []}', 400, id="Bad where filter"), pytest.param("0-8401-1070-7", '{"limit": -1}', 400, id="Bad limit filter"), - pytest.param( - "0-8401-1070-7", '{"limit": -100}', 400, id="Bad skip filter", - ), + pytest.param("0-8401-1070-7", '{"skip": -100}', 400, id="Bad skip filter"), pytest.param( "0-8401-1070-7", '{"include": ""}', 400, id="Bad include filter", ), diff --git a/test/search_api/endpoints/test_get_entity_by_pid.py b/test/search_api/endpoints/test_get_entity_by_pid.py index bbef11ea..d2cea61d 100644 --- a/test/search_api/endpoints/test_get_entity_by_pid.py +++ b/test/search_api/endpoints/test_get_entity_by_pid.py @@ -371,9 +371,7 @@ def test_valid_get_by_pid_endpoint( [ pytest.param("0-8401-1070-7", '{"where": []}', 400, id="Bad where filter"), pytest.param("0-8401-1070-7", '{"limit": -1}', 400, id="Bad limit filter"), - pytest.param( - "0-8401-1070-7", '{"limit": -100}', 400, id="Bad skip filter", - ), + pytest.param("0-8401-1070-7", '{"skip": -100}', 400, id="Bad skip filter"), pytest.param( "0-8401-1070-7", '{"include": ""}', 400, id="Bad include filter", ), diff --git a/test/search_api/endpoints/test_search_endpoint.py b/test/search_api/endpoints/test_search_endpoint.py index a5258301..024bd6e1 100644 --- a/test/search_api/endpoints/test_search_endpoint.py +++ b/test/search_api/endpoints/test_search_endpoint.py @@ -388,7 +388,7 @@ def test_valid_search_endpoint( [ pytest.param('{"where": []}', 400, id="Bad where filter"), pytest.param('{"limit": -1}', 400, id="Bad limit filter"), - pytest.param('{"limit": -100}', 400, id="Bad skip filter"), + pytest.param('{"skip": -100}', 400, id="Bad skip filter"), pytest.param('{"include": ""}', 400, id="Bad include filter"), ], ) From 0de2b5b2b713699b66164ca5732888f997230aa5 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 2 Feb 2022 14:24:12 +0000 Subject: [PATCH 15/22] fix: make get by pid endpoints return data in PaNOSC format #266 --- datagateway_api/src/search_api/helpers.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/datagateway_api/src/search_api/helpers.py b/datagateway_api/src/search_api/helpers.py index 2503c641..97bed785 100644 --- a/datagateway_api/src/search_api/helpers.py +++ b/datagateway_api/src/search_api/helpers.py @@ -2,7 +2,7 @@ import logging from datagateway_api.src.common.exceptions import MissingRecordError -from datagateway_api.src.common.filter_order_handle import FilterOrderHandler +from datagateway_api.src.common.filter_order_handler import FilterOrderHandler from datagateway_api.src.datagateway_api.icat.filters import PythonICATIncludeFilter from datagateway_api.src.search_api.filters import SearchAPIWhereFilter import datagateway_api.src.search_api.models as models @@ -87,6 +87,15 @@ def get_with_pid(entity_name, pid, filters): filters.append(SearchAPIWhereFilter("pid", pid, "eq")) + icat_relations = mappings.get_icat_relations_for_panosc_non_related_fields( + entity_name, + ) + + # Remove any duplicate ICAT relations + icat_relations = list(dict.fromkeys(icat_relations)) + if icat_relations: + filters.append(PythonICATIncludeFilter(icat_relations)) + query = SearchAPIQuery(entity_name) filter_handler = FilterOrderHandler() @@ -100,7 +109,11 @@ def get_with_pid(entity_name, pid, filters): if not icat_query_data: raise MissingRecordError("No result found") else: - return icat_query_data[0] + panosc_model = getattr(models, entity_name) + panosc_record = panosc_model.from_icat(icat_query_data[0], icat_relations).json( + by_alias=True, + ) + return json.loads(panosc_record) @client_manager From f1539db866400caa8cee4080b4c0e4e906f8af22 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 2 Feb 2022 15:32:57 +0000 Subject: [PATCH 16/22] test: fix tests after fixing bug #268 --- .../endpoints/test_get_entity_by_pid.py | 489 +++++++++--------- .../endpoints/test_search_endpoint.py | 112 +--- 2 files changed, 242 insertions(+), 359 deletions(-) diff --git a/test/search_api/endpoints/test_get_entity_by_pid.py b/test/search_api/endpoints/test_get_entity_by_pid.py index d2cea61d..de12c7fc 100644 --- a/test/search_api/endpoints/test_get_entity_by_pid.py +++ b/test/search_api/endpoints/test_get_entity_by_pid.py @@ -12,20 +12,17 @@ class TestSearchAPIGetByPIDEndpoint: "0-8401-1070-7", "{}", { - "description": "Beat professional blue clear style have. Light" - " final summer. Or hour color maybe word side much team.\nMessage" - " weight official learn especially nature. Himself tax west.", - "modTime": "2006-08-24 01:28:06+00:00", - "modId": "user", - "startDate": "2000-10-13 00:00:00+00:00", - "endDate": "2000-02-10 00:00:00+00:00", - "createId": "user", - "complete": True, - "id": 2, - "name": "DATASET 2", - "doi": "0-8401-1070-7", - "createTime": "2013-04-01 10:56:52+00:00", - "location": "/subject/break.jpeg", + "pid": "0-8401-1070-7", + "title": "DATASET 2", + "isPublic": True, + "creationDate": "2013-04-01T10:56:52+00:00", + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], }, id="Basic /datasets/{pid} request", ), @@ -34,22 +31,100 @@ class TestSearchAPIGetByPIDEndpoint: "0-449-78690-0", "{}", { - "visitId": "42", - "modId": "user", - "name": "INVESTIGATION 1", - "createId": "user", - "createTime": "2002-11-27 06:20:36+00:00", - "doi": "0-449-78690-0", - "id": 1, + "pid": "0-449-78690-0", + "isPublic": True, + "type": "INVESTIGATIONTYPE 2", + "title": "INVESTIGATION 1", "summary": "Season identify professor happen third. Beat" " professional blue clear style have. Light final summer.", - "endDate": "2000-07-09 00:00:00+00:00", - "modTime": "2005-04-30 19:41:49+00:00", - "releaseDate": "2000-07-05 00:00:00+00:00", - "startDate": "2000-04-03 00:00:00+00:00", - "title": "Including spend increase ability music skill former." - " Agreement director concern once technology sometimes someone" - " staff.\nSuccess pull bar. Laugh senior example.", + "doi": "0-449-78690-0", + "startDate": "2000-04-03T00:00:00+00:00", + "endDate": "2000-07-09T00:00:00+00:00", + "releaseDate": "2000-07-05T00:00:00+00:00", + "license": None, + "keywords": [ + "read123", + "boy129", + "out253", + "hour326", + "possible449", + "west566", + "scene948", + "who1253", + "capital1526", + "dream1989", + "front2347", + "inside2465", + "surface2851", + "learn2953", + "hot3053", + "just3159", + "population3261", + "cup3366", + "another3451", + "environmental3632", + "require3858", + "rock3952", + "determine4048", + "space4061", + "big4229", + "why4243", + "public4362", + "election4641", + "measure4996", + "often5014", + "develop5135", + "than5310", + "floor5312", + "check5327", + "cost5487", + "information6130", + "guy6180", + "admit6235", + "market6645", + "law6777", + "close7336", + "billion7597", + "product7964", + "American8041", + "language8246", + "school8277", + "specific8539", + "position8670", + "grow8702", + "time8899", + "weight9086", + "catch9129", + "speak9559", + "strong9621", + "development9757", + "best9786", + "identify10039", + "give10497", + "life10854", + "century11040", + "fire11580", + "leg11744", + "past11935", + "bar12034", + "do12108", + "prove12224", + "body12251", + "data12288", + "at12640", + "star12706", + "customer12795", + "small13058", + "event13141", + "now13193", + "magazine13415", + "policy13601", + "black13996", + "American14654", + ], + "datasets": [], + "members": [], + "parameters": [], }, id="Basic /documents/{pid} request", ), @@ -58,21 +133,10 @@ class TestSearchAPIGetByPIDEndpoint: "2", "{}", { - "description": "Former outside source play nearly Congress before" - " necessary. Allow want audience test laugh. Economic body person" - " general attorney. Effort weight prevent possible.", - "modId": "user", - "createTime": "2019-02-19 05:57:03+00:00", - "pid": None, - "createId": "user", - "type": "2", + "pid": "2", "name": "INSTRUMENT 2", - "modTime": "2019-01-29 23:33:20+00:00", - "id": 2, - "fullName": "With piece reason late model. House office fly." - " International scene call deep answer audience baby true.\n" - "Indicate education across these. Opportunity design too.", - "url": "https://moore.org/", + "facility": "LILS", + "datasets": [], }, id="Basic /instruments/{pid} request", ), @@ -80,48 +144,9 @@ class TestSearchAPIGetByPIDEndpoint: "datasets", "0-8401-1070-7", '{"include": [{"relation": "documents"}]}', - { - "createId": "user", - "startDate": "2000-10-13 00:00:00+00:00", - "doi": "0-8401-1070-7", - "modTime": "2006-08-24 01:28:06+00:00", - "createTime": "2013-04-01 10:56:52+00:00", - "location": "/subject/break.jpeg", - "endDate": "2000-02-10 00:00:00+00:00", - "complete": True, - "modId": "user", - "documents": [ - { - "createId": "user", - "doi": "0-9729806-3-6", - "startDate": "2000-06-04 00:00:00+00:00", - "modTime": "2016-11-16 19:42:34+00:00", - "createTime": "2004-08-23 02:41:19+00:00", - "endDate": "2000-09-14 00:00:00+00:00", - "modId": "user", - "title": "Show fly image herself yard challenge by. Past" - " site her number. Not weight half far move. Leader" - " everyone skin still.\nProve begin boy those always" - " dream write inside.", - "summary": "Day purpose item create. Visit hope mean admit." - " The tonight adult cut foreign would situation fund.\n" - "Purpose study usually gas think. Machine world doctor" - " rise be college treat.", - "visitId": "4", - "name": "INVESTIGATION 2", - "releaseDate": "2000-02-10 00:00:00+00:00", - "id": 2, - }, - ], - "description": "Beat professional blue clear style have. Light" - " final summer. Or hour color maybe word side much team." - "\nMessage weight official learn especially nature. Himself" - " tax west.", - "name": "DATASET 2", - "id": 2, - }, + {}, id="Get dataset by pid with include filter", - # Skipped because of incorrect document JSON format + # TODO - issue with Document.type marks=pytest.mark.skip, ), pytest.param( @@ -129,56 +154,127 @@ class TestSearchAPIGetByPIDEndpoint: "0-449-78690-0", '{"include": [{"relation": "datasets"}]}', { + "pid": "0-449-78690-0", + "isPublic": True, + "type": "INVESTIGATIONTYPE 2", + "title": "INVESTIGATION 1", + "summary": "Season identify professor happen third. Beat" + " professional blue clear style have. Light final summer.", + "doi": "0-449-78690-0", + "startDate": "2000-04-03T00:00:00+00:00", + "endDate": "2000-07-09T00:00:00+00:00", + "releaseDate": "2000-07-05T00:00:00+00:00", + "license": None, + "keywords": [ + "read123", + "boy129", + "out253", + "hour326", + "possible449", + "west566", + "scene948", + "who1253", + "capital1526", + "dream1989", + "front2347", + "inside2465", + "surface2851", + "learn2953", + "hot3053", + "just3159", + "population3261", + "cup3366", + "another3451", + "environmental3632", + "require3858", + "rock3952", + "determine4048", + "space4061", + "big4229", + "why4243", + "public4362", + "election4641", + "measure4996", + "often5014", + "develop5135", + "than5310", + "floor5312", + "check5327", + "cost5487", + "information6130", + "guy6180", + "admit6235", + "market6645", + "law6777", + "close7336", + "billion7597", + "product7964", + "American8041", + "language8246", + "school8277", + "specific8539", + "position8670", + "grow8702", + "time8899", + "weight9086", + "catch9129", + "speak9559", + "strong9621", + "development9757", + "best9786", + "identify10039", + "give10497", + "life10854", + "century11040", + "fire11580", + "leg11744", + "past11935", + "bar12034", + "do12108", + "prove12224", + "body12251", + "data12288", + "at12640", + "star12706", + "customer12795", + "small13058", + "event13141", + "now13193", + "magazine13415", + "policy13601", + "black13996", + "American14654", + ], "datasets": [ { - "createId": "user", - "startDate": "2000-05-07 00:00:00+00:00", - "doi": "0-449-78690-0", - "modTime": "2005-04-30 19:41:49+00:00", - "createTime": "2002-11-27 06:20:36+00:00", - "location": "/international/subject.tiff", - "endDate": "2000-07-05 00:00:00+00:00", - "complete": True, - "modId": "user", - "description": "Many last prepare small. Maintain throw" - " hope parent.\nEntire soon option bill fish against power." - "\nRather why rise month shake voice.", - "name": "DATASET 1", - "id": 1, + "pid": "0-449-78690-0", + "title": "DATASET 1", + "isPublic": True, + "creationDate": "2002-11-27T06:20:36+00:00", + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], }, { - "createId": "user", - "startDate": "2060-01-07 00:00:00+00:00", - "doi": "0-353-84629-5", - "modTime": "2002-09-30 13:03:32+00:00", - "createTime": "2006-11-21 17:10:42+00:00", - "location": "/gun/special.jpeg", - "endDate": "2060-01-17 00:00:00+00:00", - "complete": True, - "modId": "user", - "description": "Single many hope organization reach process" - " I. Health hit total federal describe. Bill firm rate" - " democratic outside.\nLate while our either worry.", - "name": "DATASET 241", - "id": 241, + "pid": "0-353-84629-5", + "title": "DATASET 241", + "isPublic": True, + "creationDate": "2006-11-21T17:10:42+00:00", + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], }, ], - "visitId": "42", - "modId": "user", - "name": "INVESTIGATION 1", - "createId": "user", - "createTime": "2002-11-27 06:20:36+00:00", - "doi": "0-449-78690-0", - "id": 1, - "summary": "Season identify professor happen third. Beat" - " professional blue clear style have. Light final summer.", - "endDate": "2000-07-09 00:00:00+00:00", - "modTime": "2005-04-30 19:41:49+00:00", - "releaseDate": "2000-07-05 00:00:00+00:00", - "startDate": "2000-04-03 00:00:00+00:00", - "title": "Including spend increase ability music skill former." - " Agreement director concern once technology sometimes someone" - " staff.\nSuccess pull bar. Laugh senior example.", + "members": [], + "parameters": [], }, id="Get document by pid with include filter", ), @@ -207,55 +303,26 @@ class TestSearchAPIGetByPIDEndpoint: # Skipped due to ICAT 5 mapping marks=pytest.mark.skip, ), + pytest.param( + "datasets", + "0-8401-1070-7", + '{"include": [{"relation": "documents"}, {"relation": "instrument"},' + ' {"relation": "files"}, {"relation": "parameters"},' + ' {"relation": "samples"}]}', + {}, + id="Get dataset by pid including all ICAT 4 related entities", + # TODO - issue with parameters include + marks=pytest.mark.skip, + ), pytest.param( "datasets", "0-8401-1070-7", '{"include": [{"relation": "documents"}, {"relation": "techniques"},' ' {"relation": "instrument"}, {"relation": "files"},' ' {"relation": "parameters"}, {"relation": "samples"}]}', - { - "createId": "user", - "startDate": "2000-10-13 00:00:00+00:00", - "doi": "0-8401-1070-7", - "modTime": "2006-08-24 01:28:06+00:00", - "createTime": "2013-04-01 10:56:52+00:00", - "location": "/subject/break.jpeg", - "endDate": "2000-02-10 00:00:00+00:00", - "complete": True, - "modId": "user", - "documents": [ - { - "createId": "user", - "doi": "0-9729806-3-6", - "startDate": "2000-06-04 00:00:00+00:00", - "modTime": "2016-11-16 19:42:34+00:00", - "createTime": "2004-08-23 02:41:19+00:00", - "endDate": "2000-09-14 00:00:00+00:00", - "modId": "user", - "title": "Show fly image herself yard challenge by. Past" - " site her number. Not weight half far move. Leader" - " everyone skin still.\nProve begin boy those always" - " dream write inside.", - "summary": "Day purpose item create. Visit hope mean admit." - " The tonight adult cut foreign would situation fund.\n" - "Purpose study usually gas think. Machine world doctor" - " rise be college treat.", - "visitId": "4", - "name": "INVESTIGATION 2", - "releaseDate": "2000-02-10 00:00:00+00:00", - "id": 2, - }, - ], - "description": "Beat professional blue clear style have. Light" - " final summer. Or hour color maybe word side much team." - "\nMessage weight official learn especially nature. Himself" - " tax west.", - "name": "DATASET 2", - "id": 2, - }, + {}, id="Get dataset by pid including all possible related entities", - # Skipped because of incorrect document JSON format and ICAT 5 mapping - # on techniques and instrument + # Skipped ICAT 5 mapping on techniques and instrument marks=pytest.mark.skip, ), pytest.param( @@ -263,89 +330,9 @@ class TestSearchAPIGetByPIDEndpoint: "0-449-78690-0", '{"include": [{"relation": "datasets"}, {"relation": "members"},' ' {"relation": "parameters"}]}', - { - "datasets": [ - { - "createId": "user", - "startDate": "2000-05-07 00:00:00+00:00", - "doi": "0-449-78690-0", - "modTime": "2005-04-30 19:41:49+00:00", - "createTime": "2002-11-27 06:20:36+00:00", - "location": "/international/subject.tiff", - "endDate": "2000-07-05 00:00:00+00:00", - "complete": True, - "modId": "user", - "description": "Many last prepare small. Maintain throw" - " hope parent.\nEntire soon option bill fish against power." - "\nRather why rise month shake voice.", - "parameters": [], - "name": "DATASET 1", - "id": 1, - }, - { - "createId": "user", - "startDate": "2060-01-07 00:00:00+00:00", - "doi": "0-353-84629-5", - "modTime": "2002-09-30 13:03:32+00:00", - "createTime": "2006-11-21 17:10:42+00:00", - "location": "/gun/special.jpeg", - "endDate": "2060-01-17 00:00:00+00:00", - "complete": True, - "modId": "user", - "description": "Single many hope organization reach process" - " I. Health hit total federal describe. Bill firm rate" - " democratic outside.\nLate while our either worry.", - "parameters": [], - "name": "DATASET 241", - "id": 241, - }, - ], - "createId": "user", - "doi": "0-449-78690-0", - "startDate": "2000-04-03 00:00:00+00:00", - "modTime": "2005-04-30 19:41:49+00:00", - "createTime": "2002-11-27 06:20:36+00:00", - "members": [ - { - "createId": "user", - "modId": "user", - "modTime": "2005-04-30 19:41:49+00:00", - "role": "CI", - "createTime": "2002-11-27 06:20:36+00:00", - "id": 1, - }, - ], - "endDate": "2000-07-09 00:00:00+00:00", - "modId": "user", - "title": "Including spend increase ability music skill former." - " Agreement director concern once technology sometimes someone" - " staff.\nSuccess pull bar. Laugh senior example.", - "summary": "Season identify professor happen third. Beat" - " professional blue clear style have. Light final summer.", - "visitId": "42", - "parameters": [ - { - "dateTimeValue": "2000-05-07 00:00:00+00:00", - "rangeBottom": 48.0, - "createId": "user", - "numericValue": 127265.0, - "modTime": "2005-04-30 19:41:49+00:00", - "createTime": "2002-11-27 06:20:36+00:00", - "id": 1, - "stringValue": "international1", - "modId": "user", - "rangeTop": 101.0, - "error": 31472.0, - }, - ], - "name": "INVESTIGATION 1", - "releaseDate": "2000-07-05 00:00:00+00:00", - "id": 1, - }, + {}, id="Get document by pid including all possible related entities", - # Skipped because of incorrect members JSON naming - # (investigationUsers key used instead). It's in ICAT fields too, - # not converted into PaNOSC format + # TODO - issue with parameters include marks=pytest.mark.skip, ), ], diff --git a/test/search_api/endpoints/test_search_endpoint.py b/test/search_api/endpoints/test_search_endpoint.py index 024bd6e1..aa7b606c 100644 --- a/test/search_api/endpoints/test_search_endpoint.py +++ b/test/search_api/endpoints/test_search_endpoint.py @@ -221,122 +221,18 @@ class TestSearchAPISearchEndpoint: '{"include": [{"relation": "documents"}, {"relation": "techniques"},' ' {"relation": "instrument"}, {"relation": "files"},' ' {"relation": "parameters"}, {"relation": "samples"}], "limit": 1}', - [ - { - "createId": "user", - "startDate": "2000-10-13 00:00:00+00:00", - "doi": "0-8401-1070-7", - "modTime": "2006-08-24 01:28:06+00:00", - "createTime": "2013-04-01 10:56:52+00:00", - "location": "/subject/break.jpeg", - "endDate": "2000-02-10 00:00:00+00:00", - "complete": True, - "modId": "user", - "documents": [{}], - "description": "Beat professional blue clear style have. Light" - " final summer. Or hour color maybe word side much team." - "\nMessage weight official learn especially nature. Himself" - " tax west.", - "name": "DATASET 2", - "id": 2, - }, - ], + [{}], id="Search datasets including all possible related entities", - # Skipped because of incorrect document JSON format and ICAT 5 mapping - # on techniques and instrument + # Skipped because ICAT 5 mapping on techniques and instrument marks=pytest.mark.skip, ), pytest.param( "documents", '{"include": [{"relation": "datasets"}, {"relation": "members"},' ' {"relation": "parameters"}], "limit": 1}', - [ - { - "datasets": [ - { - "createId": "user", - "startDate": "2000-05-07 00:00:00+00:00", - "doi": "0-449-78690-0", - "modTime": "2005-04-30 19:41:49+00:00", - "createTime": "2002-11-27 06:20:36+00:00", - "location": "/international/subject.tiff", - "endDate": "2000-07-05 00:00:00+00:00", - "complete": True, - "modId": "user", - "description": "Many last prepare small. Maintain throw" - " hope parent.\nEntire soon option bill fish against" - " power." - "\nRather why rise month shake voice.", - "parameters": [], - "name": "DATASET 1", - "id": 1, - }, - { - "createId": "user", - "startDate": "2060-01-07 00:00:00+00:00", - "doi": "0-353-84629-5", - "modTime": "2002-09-30 13:03:32+00:00", - "createTime": "2006-11-21 17:10:42+00:00", - "location": "/gun/special.jpeg", - "endDate": "2060-01-17 00:00:00+00:00", - "complete": True, - "modId": "user", - "description": "Single many hope organization reach" - " process I. Health hit total federal describe. Bill" - " firm rate democratic outside.\nLate while our either" - " worry.", - "parameters": [], - "name": "DATASET 241", - "id": 241, - }, - ], - "createId": "user", - "doi": "0-449-78690-0", - "startDate": "2000-04-03 00:00:00+00:00", - "modTime": "2005-04-30 19:41:49+00:00", - "createTime": "2002-11-27 06:20:36+00:00", - "members": [ - { - "createId": "user", - "modId": "user", - "modTime": "2005-04-30 19:41:49+00:00", - "role": "CI", - "createTime": "2002-11-27 06:20:36+00:00", - "id": 1, - }, - ], - "endDate": "2000-07-09 00:00:00+00:00", - "modId": "user", - "title": "Including spend increase ability music skill former." - " Agreement director concern once technology sometimes someone" - " staff.\nSuccess pull bar. Laugh senior example.", - "summary": "Season identify professor happen third. Beat" - " professional blue clear style have. Light final summer.", - "visitId": "42", - "parameters": [ - { - "dateTimeValue": "2000-05-07 00:00:00+00:00", - "rangeBottom": 48.0, - "createId": "user", - "numericValue": 127265.0, - "modTime": "2005-04-30 19:41:49+00:00", - "createTime": "2002-11-27 06:20:36+00:00", - "id": 1, - "stringValue": "international1", - "modId": "user", - "rangeTop": 101.0, - "error": 31472.0, - }, - ], - "name": "INVESTIGATION 1", - "releaseDate": "2000-07-05 00:00:00+00:00", - "id": 1, - }, - ], + [{}], id="Search documents including all possible related entities", - # Skipped because of incorrect members JSON naming - # (investigationUsers key used instead). It's in ICAT fields too, - # not converted into PaNOSC format + # TODO - issue with parameters include marks=pytest.mark.skip, ), pytest.param( From e428328e3281c4a75ee6979850db157b3efb58d1 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Wed, 2 Feb 2022 17:38:20 +0000 Subject: [PATCH 17/22] test: update pid type on endpoint rules #268 --- test/test_endpoint_rules.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/test_endpoint_rules.py b/test/test_endpoint_rules.py index f09fa004..cc439545 100644 --- a/test/test_endpoint_rules.py +++ b/test/test_endpoint_rules.py @@ -85,17 +85,17 @@ def test_entity_endpoints(self, flask_test_app, endpoint_ending, expected_method id="Search API search instruments", ), pytest.param( - f"{Config.config.search_api.extension}/datasets/", + f"{Config.config.search_api.extension}/datasets/", ["GET"], id="Search API get single dataset", ), pytest.param( - f"{Config.config.search_api.extension}/documents/", + f"{Config.config.search_api.extension}/documents/", ["GET"], id="Search API get single document", ), pytest.param( - f"{Config.config.search_api.extension}/instruments/", + f"{Config.config.search_api.extension}/instruments/", ["GET"], id="Search API get single instrument", ), @@ -115,12 +115,13 @@ def test_entity_endpoints(self, flask_test_app, endpoint_ending, expected_method id="Search API instrument count", ), pytest.param( - f"{Config.config.search_api.extension}/datasets//files", + f"{Config.config.search_api.extension}/datasets//files", ["GET"], id="Search API get dataset files", ), pytest.param( - f"{Config.config.search_api.extension}/datasets//files/count", + f"{Config.config.search_api.extension}/datasets//files" + "/count", ["GET"], id="Search API dataset files count", ), From 73b70bd756949fc23b7d4e58d6cdf5c3038a219b Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Feb 2022 13:42:21 +0000 Subject: [PATCH 18/22] refactor: make `get_with_pid()` more efficient by calling `get_search()` #266 --- datagateway_api/src/search_api/helpers.py | 28 +++-------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/datagateway_api/src/search_api/helpers.py b/datagateway_api/src/search_api/helpers.py index 97bed785..6879c8c0 100644 --- a/datagateway_api/src/search_api/helpers.py +++ b/datagateway_api/src/search_api/helpers.py @@ -87,33 +87,11 @@ def get_with_pid(entity_name, pid, filters): filters.append(SearchAPIWhereFilter("pid", pid, "eq")) - icat_relations = mappings.get_icat_relations_for_panosc_non_related_fields( - entity_name, - ) - - # Remove any duplicate ICAT relations - icat_relations = list(dict.fromkeys(icat_relations)) - if icat_relations: - filters.append(PythonICATIncludeFilter(icat_relations)) - - query = SearchAPIQuery(entity_name) - - filter_handler = FilterOrderHandler() - filter_handler.add_filters(filters) - filter_handler.merge_python_icat_limit_skip_filters() - filter_handler.apply_filters(query) - - log.debug("JPQL Query to be sent/executed in ICAT: %s", query.icat_query.query) - icat_query_data = query.icat_query.execute_query(SessionHandler.client, True) - - if not icat_query_data: + panosc_data = get_search(entity_name, filters) + if not panosc_data: raise MissingRecordError("No result found") else: - panosc_model = getattr(models, entity_name) - panosc_record = panosc_model.from_icat(icat_query_data[0], icat_relations).json( - by_alias=True, - ) - return json.loads(panosc_record) + return panosc_data[0] @client_manager From 8840a6cfe4134421d60fa2acdf329d4d737e151c Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Feb 2022 13:44:24 +0000 Subject: [PATCH 19/22] refactor: move ICAT relations call to filter handler #268 --- .../src/common/filter_order_handler.py | 23 +++++++++++++++++++ datagateway_api/src/search_api/helpers.py | 16 +++++-------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/datagateway_api/src/common/filter_order_handler.py b/datagateway_api/src/common/filter_order_handler.py index 2ce9b03e..db9a5f87 100644 --- a/datagateway_api/src/common/filter_order_handler.py +++ b/datagateway_api/src/common/filter_order_handler.py @@ -108,6 +108,29 @@ def add_icat_relations_for_non_related_fields_of_panosc_related_entities( python_icat_include_filter = PythonICATIncludeFilter(icat_relations) self.filters.append(python_icat_include_filter) + def add_icat_relations_for_panosc_non_related_fields( + self, panosc_entity_name, + ): + """ + Retrieve ICAT relations and create a `PythonICATIncludeFilter` for these ICAT + relations + + :param panosc_entity_name: A PaNOSC entity name e.g. "Dataset" + :type panosc_entity_name: :class:`str` + :return: ICAT relations for the non related fields of the given PaNOSC entity + """ + + icat_relations = mappings.get_icat_relations_for_panosc_non_related_fields( + panosc_entity_name, + ) + + # Remove any duplicate ICAT relations + icat_relations = list(dict.fromkeys(icat_relations)) + if icat_relations: + self.filters.append(PythonICATIncludeFilter(icat_relations)) + + return icat_relations + def merge_python_icat_limit_skip_filters(self): """ When there are both limit and skip filters in a request, merge them into the diff --git a/datagateway_api/src/search_api/helpers.py b/datagateway_api/src/search_api/helpers.py index 6879c8c0..e6b17805 100644 --- a/datagateway_api/src/search_api/helpers.py +++ b/datagateway_api/src/search_api/helpers.py @@ -3,10 +3,8 @@ from datagateway_api.src.common.exceptions import MissingRecordError from datagateway_api.src.common.filter_order_handler import FilterOrderHandler -from datagateway_api.src.datagateway_api.icat.filters import PythonICATIncludeFilter from datagateway_api.src.search_api.filters import SearchAPIWhereFilter import datagateway_api.src.search_api.models as models -from datagateway_api.src.search_api.panosc_mappings import mappings from datagateway_api.src.search_api.query import SearchAPIQuery from datagateway_api.src.search_api.session_handler import ( client_manager, @@ -34,18 +32,16 @@ def get_search(entity_name, filters): log.info("Searching for %s using request's filters", entity_name) log.debug("Entity Name: %s, Filters: %s", entity_name, filters) - icat_relations = mappings.get_icat_relations_for_panosc_non_related_fields( - entity_name, - ) - # Remove any duplicate ICAT relations - icat_relations = list(dict.fromkeys(icat_relations)) - if icat_relations: - filters.append(PythonICATIncludeFilter(icat_relations)) - query = SearchAPIQuery(entity_name) filter_handler = FilterOrderHandler() filter_handler.add_filters(filters) + icat_relations = filter_handler.add_icat_relations_for_panosc_non_related_fields( + entity_name, + ) + filter_handler.add_icat_relations_for_non_related_fields_of_panosc_related_entities( + entity_name, + ) filter_handler.merge_python_icat_limit_skip_filters() filter_handler.apply_filters(query) From de2630327bdc9806409cb75871e9712cbe086a2e Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Feb 2022 13:51:12 +0000 Subject: [PATCH 20/22] test: unskip endpoint tests and inject data #266, #268 --- .../endpoints/test_get_entity_by_pid.py | 254 +++++++++++++++++- .../endpoints/test_search_endpoint.py | 145 +++++++++- 2 files changed, 388 insertions(+), 11 deletions(-) diff --git a/test/search_api/endpoints/test_get_entity_by_pid.py b/test/search_api/endpoints/test_get_entity_by_pid.py index de12c7fc..cb656bf5 100644 --- a/test/search_api/endpoints/test_get_entity_by_pid.py +++ b/test/search_api/endpoints/test_get_entity_by_pid.py @@ -144,10 +144,111 @@ class TestSearchAPIGetByPIDEndpoint: "datasets", "0-8401-1070-7", '{"include": [{"relation": "documents"}]}', - {}, + { + "pid": "0-8401-1070-7", + "title": "DATASET 2", + "isPublic": True, + "creationDate": "2013-04-01T10:56:52+00:00", + "size": None, + "documents": [ + { + "pid": "0-9729806-3-6", + "isPublic": True, + "type": "INVESTIGATIONTYPE 1", + "title": "INVESTIGATION 2", + "summary": "Day purpose item create. Visit hope mean admit." + " The tonight adult cut foreign would situation fund." + "\nPurpose study usually gas think. Machine world doctor" + " rise be college treat.", + "doi": "0-9729806-3-6", + "startDate": "2000-06-04T00:00:00+00:00", + "endDate": "2000-09-14T00:00:00+00:00", + "releaseDate": "2000-02-10T00:00:00+00:00", + "license": None, + "keywords": [ + "later868", + "black887", + "yard993", + "as1034", + "sister1072", + "knowledge1239", + "heart1286", + "soon1911", + "television2011", + "turn2206", + "PM2477", + "purpose2641", + "box3225", + "fall3339", + "minute3394", + "family3729", + "industry3903", + "throw4294", + "here4636", + "firm4658", + "same4777", + "travel5029", + "like5061", + "but5204", + "pass5300", + "industry5482", + "they6191", + "company6197", + "bed6368", + "shake6405", + "voice6555", + "above6582", + "former6596", + "body6759", + "price6828", + "star6879", + "clearly7066", + "wall7234", + "ground7790", + "maybe8191", + "population8212", + "indeed8287", + "decide8366", + "agree9237", + "color9257", + "once9593", + "two9737", + "they10312", + "heavy10342", + "decide10450", + "main10579", + "particular10646", + "number11411", + "investment11462", + "interesting11775", + "simple12151", + "stand12178", + "morning12276", + "fish12494", + "story12572", + "art13071", + "once13596", + "music13672", + "film13812", + "level14266", + "claim14304", + "nearly14473", + "eight14566", + "president14766", + "near14993", + ], + "datasets": [], + "members": [], + "parameters": [], + }, + ], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], + }, id="Get dataset by pid with include filter", - # TODO - issue with Document.type - marks=pytest.mark.skip, ), pytest.param( "documents", @@ -306,12 +407,12 @@ class TestSearchAPIGetByPIDEndpoint: pytest.param( "datasets", "0-8401-1070-7", - '{"include": [{"relation": "documents"}, {"relation": "instrument"},' + '{"include": [{"relation": "documents"},' ' {"relation": "files"}, {"relation": "parameters"},' ' {"relation": "samples"}]}', {}, id="Get dataset by pid including all ICAT 4 related entities", - # TODO - issue with parameters include + # Skipped due to #314 (Sample.pid) marks=pytest.mark.skip, ), pytest.param( @@ -330,10 +431,147 @@ class TestSearchAPIGetByPIDEndpoint: "0-449-78690-0", '{"include": [{"relation": "datasets"}, {"relation": "members"},' ' {"relation": "parameters"}]}', - {}, + { + "pid": "0-449-78690-0", + "isPublic": True, + "type": "INVESTIGATIONTYPE 2", + "title": "INVESTIGATION 1", + "summary": "Season identify professor happen third. Beat" + " professional blue clear style have. Light final summer.", + "doi": "0-449-78690-0", + "startDate": "2000-04-03T00:00:00+00:00", + "endDate": "2000-07-09T00:00:00+00:00", + "releaseDate": "2000-07-05T00:00:00+00:00", + "license": None, + "keywords": [ + "read123", + "boy129", + "out253", + "hour326", + "possible449", + "west566", + "scene948", + "who1253", + "capital1526", + "dream1989", + "front2347", + "inside2465", + "surface2851", + "learn2953", + "hot3053", + "just3159", + "population3261", + "cup3366", + "another3451", + "environmental3632", + "require3858", + "rock3952", + "determine4048", + "space4061", + "big4229", + "why4243", + "public4362", + "election4641", + "measure4996", + "often5014", + "develop5135", + "than5310", + "floor5312", + "check5327", + "cost5487", + "information6130", + "guy6180", + "admit6235", + "market6645", + "law6777", + "close7336", + "billion7597", + "product7964", + "American8041", + "language8246", + "school8277", + "specific8539", + "position8670", + "grow8702", + "time8899", + "weight9086", + "catch9129", + "speak9559", + "strong9621", + "development9757", + "best9786", + "identify10039", + "give10497", + "life10854", + "century11040", + "fire11580", + "leg11744", + "past11935", + "bar12034", + "do12108", + "prove12224", + "body12251", + "data12288", + "at12640", + "star12706", + "customer12795", + "small13058", + "event13141", + "now13193", + "magazine13415", + "policy13601", + "black13996", + "American14654", + ], + "datasets": [ + { + "pid": "0-449-78690-0", + "title": "DATASET 1", + "isPublic": True, + "creationDate": "2002-11-27T06:20:36+00:00", + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], + }, + { + "pid": "0-353-84629-5", + "title": "DATASET 241", + "isPublic": True, + "creationDate": "2006-11-21T17:10:42+00:00", + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], + }, + ], + "members": [ + { + "id": "1", + "role": "CI", + "document": None, + "person": None, + "affiliation": None, + }, + ], + "parameters": [ + { + "id": "1", + "name": "PARAMETERTYPE 27", + "value": 127265.0, + "unit": "unit 27", + "dataset": None, + "document": None, + }, + ], + }, id="Get document by pid including all possible related entities", - # TODO - issue with parameters include - marks=pytest.mark.skip, ), ], ) diff --git a/test/search_api/endpoints/test_search_endpoint.py b/test/search_api/endpoints/test_search_endpoint.py index aa7b606c..a89ffd3b 100644 --- a/test/search_api/endpoints/test_search_endpoint.py +++ b/test/search_api/endpoints/test_search_endpoint.py @@ -230,10 +230,149 @@ class TestSearchAPISearchEndpoint: "documents", '{"include": [{"relation": "datasets"}, {"relation": "members"},' ' {"relation": "parameters"}], "limit": 1}', - [{}], + [ + { + "pid": "0-449-78690-0", + "isPublic": True, + "type": "INVESTIGATIONTYPE 2", + "title": "INVESTIGATION 1", + "summary": "Season identify professor happen third. Beat" + " professional blue clear style have. Light final summer.", + "doi": "0-449-78690-0", + "startDate": "2000-04-03T00:00:00+00:00", + "endDate": "2000-07-09T00:00:00+00:00", + "releaseDate": "2000-07-05T00:00:00+00:00", + "license": None, + "keywords": [ + "read123", + "boy129", + "out253", + "hour326", + "possible449", + "west566", + "scene948", + "who1253", + "capital1526", + "dream1989", + "front2347", + "inside2465", + "surface2851", + "learn2953", + "hot3053", + "just3159", + "population3261", + "cup3366", + "another3451", + "environmental3632", + "require3858", + "rock3952", + "determine4048", + "space4061", + "big4229", + "why4243", + "public4362", + "election4641", + "measure4996", + "often5014", + "develop5135", + "than5310", + "floor5312", + "check5327", + "cost5487", + "information6130", + "guy6180", + "admit6235", + "market6645", + "law6777", + "close7336", + "billion7597", + "product7964", + "American8041", + "language8246", + "school8277", + "specific8539", + "position8670", + "grow8702", + "time8899", + "weight9086", + "catch9129", + "speak9559", + "strong9621", + "development9757", + "best9786", + "identify10039", + "give10497", + "life10854", + "century11040", + "fire11580", + "leg11744", + "past11935", + "bar12034", + "do12108", + "prove12224", + "body12251", + "data12288", + "at12640", + "star12706", + "customer12795", + "small13058", + "event13141", + "now13193", + "magazine13415", + "policy13601", + "black13996", + "American14654", + ], + "datasets": [ + { + "pid": "0-449-78690-0", + "title": "DATASET 1", + "isPublic": True, + "creationDate": "2002-11-27T06:20:36+00:00", + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], + }, + { + "pid": "0-353-84629-5", + "title": "DATASET 241", + "isPublic": True, + "creationDate": "2006-11-21T17:10:42+00:00", + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], + }, + ], + "members": [ + { + "id": "1", + "role": "CI", + "document": None, + "person": None, + "affiliation": None, + }, + ], + "parameters": [ + { + "id": "1", + "name": "PARAMETERTYPE 27", + "value": 127265.0, + "unit": "unit 27", + "dataset": None, + "document": None, + }, + ], + }, + ], id="Search documents including all possible related entities", - # TODO - issue with parameters include - marks=pytest.mark.skip, ), pytest.param( "instruments", From 22a0ea850435d0f4ca4326b7525c5a0fa1d1d8a5 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Thu, 3 Feb 2022 16:58:30 +0000 Subject: [PATCH 21/22] ci: add anon user to ICAT Server rootUserNames #268 --- .github/workflows/ci-build.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 1d9a4a47..ddcff9d1 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -59,6 +59,17 @@ jobs: run: | ansible-playbook icat-ansible/icatdb-minimal-hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv + # rootUserNames needs editing as anon/anon is used in search API and required to pass endpoint tests + - name: Add anon user to rootUserNames + run: | + awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp + - name: Apply rootUserNames change + run: | + mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties + - name: Reinstall ICAT Server + run: | + cd /home/runner/install/icat.server/ && ./setup -vv install + - name: Checkout DataGateway API uses: actions/checkout@v2 From 4343a2c67cdba63a0b21e552b7d630eaf0b46cd6 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Mon, 7 Feb 2022 12:00:21 +0000 Subject: [PATCH 22/22] refactor: refactor entity relations code #268 --- .../src/common/filter_order_handler.py | 3 --- datagateway_api/src/search_api/helpers.py | 16 +++++++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/datagateway_api/src/common/filter_order_handler.py b/datagateway_api/src/common/filter_order_handler.py index db9a5f87..dfe36530 100644 --- a/datagateway_api/src/common/filter_order_handler.py +++ b/datagateway_api/src/common/filter_order_handler.py @@ -117,7 +117,6 @@ def add_icat_relations_for_panosc_non_related_fields( :param panosc_entity_name: A PaNOSC entity name e.g. "Dataset" :type panosc_entity_name: :class:`str` - :return: ICAT relations for the non related fields of the given PaNOSC entity """ icat_relations = mappings.get_icat_relations_for_panosc_non_related_fields( @@ -129,8 +128,6 @@ def add_icat_relations_for_panosc_non_related_fields( if icat_relations: self.filters.append(PythonICATIncludeFilter(icat_relations)) - return icat_relations - def merge_python_icat_limit_skip_filters(self): """ When there are both limit and skip filters in a request, merge them into the diff --git a/datagateway_api/src/search_api/helpers.py b/datagateway_api/src/search_api/helpers.py index e6b17805..43513174 100644 --- a/datagateway_api/src/search_api/helpers.py +++ b/datagateway_api/src/search_api/helpers.py @@ -3,7 +3,10 @@ from datagateway_api.src.common.exceptions import MissingRecordError from datagateway_api.src.common.filter_order_handler import FilterOrderHandler -from datagateway_api.src.search_api.filters import SearchAPIWhereFilter +from datagateway_api.src.search_api.filters import ( + SearchAPIIncludeFilter, + SearchAPIWhereFilter, +) import datagateway_api.src.search_api.models as models from datagateway_api.src.search_api.query import SearchAPIQuery from datagateway_api.src.search_api.session_handler import ( @@ -32,13 +35,16 @@ def get_search(entity_name, filters): log.info("Searching for %s using request's filters", entity_name) log.debug("Entity Name: %s, Filters: %s", entity_name, filters) + entity_relations = [] + for filter_ in filters: + if isinstance(filter_, SearchAPIIncludeFilter): + entity_relations.extend(filter_.included_filters) + query = SearchAPIQuery(entity_name) filter_handler = FilterOrderHandler() filter_handler.add_filters(filters) - icat_relations = filter_handler.add_icat_relations_for_panosc_non_related_fields( - entity_name, - ) + filter_handler.add_icat_relations_for_panosc_non_related_fields(entity_name) filter_handler.add_icat_relations_for_non_related_fields_of_panosc_related_entities( entity_name, ) @@ -51,7 +57,7 @@ def get_search(entity_name, filters): panosc_data = [] for icat_data in icat_query_data: panosc_model = getattr(models, entity_name) - panosc_record = panosc_model.from_icat(icat_data, icat_relations).json( + panosc_record = panosc_model.from_icat(icat_data, entity_relations).json( by_alias=True, ) panosc_data.append(json.loads(panosc_record))