Merge pull request #325 from ral-facilities/make-num-years-determinin…

…g-public-data-configurable-#312

Make Number of Years to Determine Public Data Configurable

datagateway_api/config.json.example

@@ -12,7 +12,8 @@
     "search_api": {
         "extension": "/search-api",
         "icat_url": "https://localhost:8181",
-        "icat_check_cert": false
+        "icat_check_cert": false,
+        "num_of_years_determining_public_data": 3
     },
     "flask_reloader": false,
     "log_level": "WARN",

datagateway_api/src/common/config.py

@@ -130,6 +130,7 @@ class SearchAPI(BaseModel):
     extension: StrictStr
     icat_check_cert: StrictBool
     icat_url: StrictStr
+    num_of_years_determining_public_data: StrictInt
 
     _validate_extension = validator("extension", allow_reuse=True)(validate_extension)
 

datagateway_api/src/search_api/models.py

@@ -7,6 +7,7 @@
 from pydantic import BaseModel, Field, ValidationError, validator
 from pydantic.error_wrappers import ErrorWrapper
 
+from datagateway_api.src.common.config import Config
 from datagateway_api.src.common.date_handler import DateHandler
 from datagateway_api.src.search_api.panosc_mappings import mappings
 
@@ -197,8 +198,10 @@ def set_is_public(cls, value):  # noqa: B902, N805
 
         creation_date = DateHandler.str_to_datetime_object(value)
         current_datetime = datetime.now(timezone.utc)
-        three_years_ago = current_datetime - relativedelta(years=3)
-        return creation_date < three_years_ago
+        rd = relativedelta(
+            years=Config.config.search_api.num_of_years_determining_public_data,
+        )
+        return creation_date < (current_datetime - rd)
 
     @classmethod
     def from_icat(cls, icat_data, required_related_fields):
@@ -236,8 +239,10 @@ def set_is_public(cls, value):  # noqa: B902, N805
 
         creation_date = DateHandler.str_to_datetime_object(value)
         current_datetime = datetime.now(timezone.utc)
-        three_years_ago = current_datetime - relativedelta(years=3)
-        return creation_date < three_years_ago
+        rd = relativedelta(
+            years=Config.config.search_api.num_of_years_determining_public_data,
+        )
+        return creation_date < (current_datetime - rd)
 
     @classmethod
     def from_icat(cls, icat_data, required_related_fields):

datagateway_api/src/search_api/query_filter_factory.py

@@ -4,6 +4,7 @@
 from dateutil.relativedelta import relativedelta
 
 from datagateway_api.src.common.base_query_filter_factory import QueryFilterFactory
+from datagateway_api.src.common.config import Config
 from datagateway_api.src.common.exceptions import FilterError, SearchAPIError
 from datagateway_api.src.search_api.filters import (
     SearchAPIIncludeFilter,
@@ -361,16 +362,18 @@ def convert_is_public_field_value_and_operation(value, operation):
         so that all Datasets older than 3 years (which ISIS considers public) are
         returned.
         """
-        current_datetime = datetime.now(timezone.utc)
-        three_years_ago = current_datetime - relativedelta(years=3)
         value = not value if operation == "neq" else value
         if value is True:
             operation = "lt"
         else:
             operation = "gt"
 
+        current_datetime = datetime.now(timezone.utc)
+        rd = relativedelta(
+            years=Config.config.search_api.num_of_years_determining_public_data,
+        )
         # The timezone part has a plus sign so replacing
         # with a blank space to avoid issues
-        value = str(three_years_ago).replace("+", " ")
+        value = str(current_datetime - rd).replace("+", " ")
 
         return value, operation

test/conftest.py

@@ -104,6 +104,7 @@ def test_config_data():
             "extension": "/search-api",
             "icat_url": "https://localhost.testdomain:8181",
             "icat_check_cert": True,
+            "num_of_years_determining_public_data": 3,
         },
         "flask_reloader": False,
         "log_level": "WARN",

test/search_api/endpoints/test_count_endpoint.py

@@ -1,6 +1,9 @@
+from unittest.mock import patch
+
 import pytest
 
 from datagateway_api.src.common.config import Config
+from datagateway_api.src.common.date_handler import DateHandler
 
 
 class TestSearchAPICountEndpoint:
@@ -67,6 +70,28 @@ class TestSearchAPICountEndpoint:
                 {"count": 13},
                 id="Instrument count with where (operator specified)",
             ),
+            pytest.param(
+                "instruments",
+                '{"facility": {"like": "LILS"}}',
+                {"count": 14},
+                id="Instrument count with where using related ICAT mapping",
+            ),
+        ],
+    )
+    def test_valid_count_endpoint(
+        self, flask_test_app_search_api, endpoint_name, request_filter, expected_json,
+    ):
+        test_response = flask_test_app_search_api.get(
+            f"{Config.config.search_api.extension}/{endpoint_name}/count?where="
+            f"{request_filter}",
+        )
+
+        assert test_response.status_code == 200
+        assert test_response.json == expected_json
+
+    @pytest.mark.parametrize(
+        "endpoint_name, request_filter, expected_json",
+        [
             pytest.param(
                 "datasets",
                 '{"isPublic": true}',
@@ -79,17 +104,29 @@ class TestSearchAPICountEndpoint:
                 {"count": 76},
                 id="Document count with isPublic condition",
             ),
-            pytest.param(
-                "instruments",
-                '{"facility": {"like": "LILS"}}',
-                {"count": 14},
-                id="Instrument count with where using related ICAT mapping",
-            ),
         ],
     )
-    def test_valid_count_endpoint(
-        self, flask_test_app_search_api, endpoint_name, request_filter, expected_json,
+    @patch("datagateway_api.src.search_api.query_filter_factory.datetime")
+    def test_valid_count_endpoint_is_public_field(
+        self,
+        datetime_mock,
+        flask_test_app_search_api,
+        endpoint_name,
+        request_filter,
+        expected_json,
     ):
+        """
+        The datetime must be mocked here to prevent tests from failing as time passes.
+        A dataset or document that was created or released 2 years and 364 ago would be
+        fall in the not public category, however that same dataset or document would
+        fall in the public category (in the case of ISIS) a few days later because it
+        will be 3 years old. As a result of this, the tests will fail because the actual
+        count will be different to that of the expected. Mocking datetime takes care of
+        this issue because it sets the time to the one provided in this test method.
+        """
+        datetime_mock.now.return_value = DateHandler.str_to_datetime_object(
+            "2022-02-06 00:00:01+00:00",
+        )
         test_response = flask_test_app_search_api.get(
             f"{Config.config.search_api.extension}/{endpoint_name}/count?where="
             f"{request_filter}",