From 58e777b5c4a562f6945adcd1b55ce1d470f5d816 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Tue, 8 Feb 2022 09:24:26 +0000 Subject: [PATCH 1/3] feat(config)!: add configuration option for determining public data #312 --- datagateway_api/config.json.example | 3 ++- datagateway_api/src/common/config.py | 1 + test/conftest.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/datagateway_api/config.json.example b/datagateway_api/config.json.example index 3cce6d58..2bf972a8 100644 --- a/datagateway_api/config.json.example +++ b/datagateway_api/config.json.example @@ -12,7 +12,8 @@ "search_api": { "extension": "/search-api", "icat_url": "https://localhost:8181", - "icat_check_cert": false + "icat_check_cert": false, + "num_of_years_determining_public_data": 3 }, "flask_reloader": false, "log_level": "WARN", diff --git a/datagateway_api/src/common/config.py b/datagateway_api/src/common/config.py index e4835cd3..288f47af 100644 --- a/datagateway_api/src/common/config.py +++ b/datagateway_api/src/common/config.py @@ -130,6 +130,7 @@ class SearchAPI(BaseModel): extension: StrictStr icat_check_cert: StrictBool icat_url: StrictStr + num_of_years_determining_public_data: StrictInt _validate_extension = validator("extension", allow_reuse=True)(validate_extension) diff --git a/test/conftest.py b/test/conftest.py index 9cc80041..41e5420a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -104,6 +104,7 @@ def test_config_data(): "extension": "/search-api", "icat_url": "https://localhost.testdomain:8181", "icat_check_cert": True, + "num_of_years_determining_public_data": 3, }, "flask_reloader": False, "log_level": "WARN", From 1021c802f84e298a8a61dc6563804ce8a6eade72 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Tue, 8 Feb 2022 09:42:14 +0000 Subject: [PATCH 2/3] determine public data based on config value #312 --- datagateway_api/src/search_api/models.py | 13 +++++++++---- .../src/search_api/query_filter_factory.py | 9 ++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/datagateway_api/src/search_api/models.py b/datagateway_api/src/search_api/models.py index f9c8ae9a..0ce3187a 100644 --- a/datagateway_api/src/search_api/models.py +++ b/datagateway_api/src/search_api/models.py @@ -7,6 +7,7 @@ from pydantic import BaseModel, Field, ValidationError, validator from pydantic.error_wrappers import ErrorWrapper +from datagateway_api.src.common.config import Config from datagateway_api.src.common.date_handler import DateHandler from datagateway_api.src.search_api.panosc_mappings import mappings @@ -197,8 +198,10 @@ def set_is_public(cls, value): # noqa: B902, N805 creation_date = DateHandler.str_to_datetime_object(value) current_datetime = datetime.now(timezone.utc) - three_years_ago = current_datetime - relativedelta(years=3) - return creation_date < three_years_ago + rd = relativedelta( + years=Config.config.search_api.num_of_years_determining_public_data, + ) + return creation_date < (current_datetime - rd) @classmethod def from_icat(cls, icat_data, required_related_fields): @@ -236,8 +239,10 @@ def set_is_public(cls, value): # noqa: B902, N805 creation_date = DateHandler.str_to_datetime_object(value) current_datetime = datetime.now(timezone.utc) - three_years_ago = current_datetime - relativedelta(years=3) - return creation_date < three_years_ago + rd = relativedelta( + years=Config.config.search_api.num_of_years_determining_public_data, + ) + return creation_date < (current_datetime - rd) @classmethod def from_icat(cls, icat_data, required_related_fields): diff --git a/datagateway_api/src/search_api/query_filter_factory.py b/datagateway_api/src/search_api/query_filter_factory.py index f0dd05d2..0086b7d1 100644 --- a/datagateway_api/src/search_api/query_filter_factory.py +++ b/datagateway_api/src/search_api/query_filter_factory.py @@ -4,6 +4,7 @@ from dateutil.relativedelta import relativedelta from datagateway_api.src.common.base_query_filter_factory import QueryFilterFactory +from datagateway_api.src.common.config import Config from datagateway_api.src.common.exceptions import FilterError, SearchAPIError from datagateway_api.src.search_api.filters import ( SearchAPIIncludeFilter, @@ -361,16 +362,18 @@ def convert_is_public_field_value_and_operation(value, operation): so that all Datasets older than 3 years (which ISIS considers public) are returned. """ - current_datetime = datetime.now(timezone.utc) - three_years_ago = current_datetime - relativedelta(years=3) value = not value if operation == "neq" else value if value is True: operation = "lt" else: operation = "gt" + current_datetime = datetime.now(timezone.utc) + rd = relativedelta( + years=Config.config.search_api.num_of_years_determining_public_data, + ) # The timezone part has a plus sign so replacing # with a blank space to avoid issues - value = str(three_years_ago).replace("+", " ") + value = str(current_datetime - rd).replace("+", " ") return value, operation From 20e3cb1445e62a35a44e175ec578358ef3b5b769 Mon Sep 17 00:00:00 2001 From: Viktor Bozhinov Date: Tue, 8 Feb 2022 09:44:06 +0000 Subject: [PATCH 3/3] test: fix failing count with `isPublic` condition tests #312 --- .../endpoints/test_count_endpoint.py | 53 ++++++++++++++++--- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/test/search_api/endpoints/test_count_endpoint.py b/test/search_api/endpoints/test_count_endpoint.py index 2e41d18c..5942c122 100644 --- a/test/search_api/endpoints/test_count_endpoint.py +++ b/test/search_api/endpoints/test_count_endpoint.py @@ -1,6 +1,9 @@ +from unittest.mock import patch + import pytest from datagateway_api.src.common.config import Config +from datagateway_api.src.common.date_handler import DateHandler class TestSearchAPICountEndpoint: @@ -67,6 +70,28 @@ class TestSearchAPICountEndpoint: {"count": 13}, id="Instrument count with where (operator specified)", ), + pytest.param( + "instruments", + '{"facility": {"like": "LILS"}}', + {"count": 14}, + id="Instrument count with where using related ICAT mapping", + ), + ], + ) + def test_valid_count_endpoint( + self, flask_test_app_search_api, endpoint_name, request_filter, expected_json, + ): + test_response = flask_test_app_search_api.get( + f"{Config.config.search_api.extension}/{endpoint_name}/count?where=" + f"{request_filter}", + ) + + assert test_response.status_code == 200 + assert test_response.json == expected_json + + @pytest.mark.parametrize( + "endpoint_name, request_filter, expected_json", + [ pytest.param( "datasets", '{"isPublic": true}', @@ -79,17 +104,29 @@ class TestSearchAPICountEndpoint: {"count": 76}, id="Document count with isPublic condition", ), - pytest.param( - "instruments", - '{"facility": {"like": "LILS"}}', - {"count": 14}, - id="Instrument count with where using related ICAT mapping", - ), ], ) - def test_valid_count_endpoint( - self, flask_test_app_search_api, endpoint_name, request_filter, expected_json, + @patch("datagateway_api.src.search_api.query_filter_factory.datetime") + def test_valid_count_endpoint_is_public_field( + self, + datetime_mock, + flask_test_app_search_api, + endpoint_name, + request_filter, + expected_json, ): + """ + The datetime must be mocked here to prevent tests from failing as time passes. + A dataset or document that was created or released 2 years and 364 ago would be + fall in the not public category, however that same dataset or document would + fall in the public category (in the case of ISIS) a few days later because it + will be 3 years old. As a result of this, the tests will fail because the actual + count will be different to that of the expected. Mocking datetime takes care of + this issue because it sets the time to the one provided in this test method. + """ + datetime_mock.now.return_value = DateHandler.str_to_datetime_object( + "2022-02-06 00:00:01+00:00", + ) test_response = flask_test_app_search_api.get( f"{Config.config.search_api.extension}/{endpoint_name}/count?where=" f"{request_filter}",