Merge branch 'master' into Allow-the-configuration-of-authentication-…

…mechanisms-#350

.github/workflows/ci-build.yml

@@ -41,7 +41,7 @@ jobs:
 
       # Prep for running the playbook
       - name: Create hosts file
-        run: echo -e "[icatdb-minimal-hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
+        run: echo -e "[icatdb_minimal_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
       - name: Prepare vault pass
         run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
       - name: Move vault to directory it'll get detected by Ansible
@@ -54,10 +54,17 @@ jobs:
       - name: Change hostname to localhost
         run: sudo hostname -b localhost
 
+      # Remove existing MySQL installation so it doesn't interfere with GitHub Actions
+      - name: Remove existing mysql
+        run: |
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+          sudo apt-get remove --purge "mysql*"
+          sudo rm -rf /var/lib/mysql* /etc/mysql
+
       # Create local instance of ICAT
       - name: Run ICAT Ansible Playbook
         run: |
-          ansible-playbook icat-ansible/icatdb-minimal-hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
+          ansible-playbook icat-ansible/icatdb_minimal_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
 
       # rootUserNames needs editing as anon/anon is used in search API and required to pass endpoint tests
       - name: Add anon user to rootUserNames
@@ -202,7 +209,7 @@ jobs:
 
       # Prep for running the playbook
       - name: Create hosts file
-        run: echo -e "[icatdb-minimal-hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
+        run: echo -e "[icatdb_minimal_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
       - name: Prepare vault pass
         run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
       - name: Move vault to directory it'll get detected by Ansible
@@ -215,10 +222,17 @@ jobs:
       - name: Change hostname to localhost
         run: sudo hostname -b localhost
 
+      # Remove existing MySQL installation so it doesn't interfere with GitHub Actions
+      - name: Remove existing mysql
+        run: |
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+          sudo apt-get remove --purge "mysql*"
+          sudo rm -rf /var/lib/mysql* /etc/mysql
+
       # Create local instance of ICAT
       - name: Run ICAT Ansible Playbook
         run: |
-          ansible-playbook icat-ansible/icatdb-minimal-hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
+          ansible-playbook icat-ansible/icatdb_minimal_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
 
       - name: Checkout DataGateway API
         uses: actions/checkout@v2

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 <!--next-version-placeholder-->
 
+## v4.3.0 (2022-03-18)
+### Feature
+* Allow multiple datetime formats to be used when filtering in search API #338 ([`f20ad24`](https://github.com/ral-facilities/datagateway-api/commit/f20ad24886f85bb3ba70abfd29352b1a7e5c58ff))
+
+### Fix
+* Output datetime data in the same format as SciCat #338 ([`4596db9`](https://github.com/ral-facilities/datagateway-api/commit/4596db98808f35488aca86667aa26811d777b58e))
+
 ## v4.2.0 (2022-02-28)
 ### Feature
 * Create openapi endpoint for Search API #281 ([`412458c`](https://github.com/ral-facilities/datagateway-api/commit/412458cc4cc73230db0115bdbfdfe6ac815d42c1))

datagateway_api/src/search_api/filters.py

@@ -9,6 +9,7 @@
     PythonICATSkipFilter,
     PythonICATWhereFilter,
 )
+from datagateway_api.src.search_api.models import PaNOSCAttribute
 from datagateway_api.src.search_api.panosc_mappings import mappings
 from datagateway_api.src.search_api.query import SearchAPIQuery
 
@@ -19,6 +20,19 @@ class SearchAPIWhereFilter(PythonICATWhereFilter):
     def __init__(self, field, value, operation, search_api_query=None):
         self.search_api_query = search_api_query
         super().__init__(field, value, operation)
+
+        # Detect various datetime formats and convert them into a format that ICAT can
+        # understand
+        if (
+            self.field in PaNOSCAttribute._datetime_field_names
+            and isinstance(self.value, str)
+            and DateHandler.is_str_a_date(self.value)
+        ):
+            value_datetime = DateHandler.str_to_datetime_object(value)
+            str_datetime = DateHandler.datetime_object_to_str(value_datetime)
+            # +/- need to be removed so the format works when querying ICAT
+            self.value = str_datetime.replace("+", " ").replace("-", " ")
+
         log.info("SearchAPIWhereFilter created: %s", repr(self))
 
     def apply_filter(self, query):

datagateway_api/src/search_api/models.py

@@ -4,6 +4,7 @@
 from typing import ClassVar, List, Optional, Union
 
 from pydantic import BaseModel, Field, root_validator, ValidationError, validator
+from pydantic.datetime_parse import parse_datetime
 from pydantic.error_wrappers import ErrorWrapper
 
 from datagateway_api.src.search_api.panosc_mappings import mappings
@@ -44,7 +45,26 @@ def _get_icat_field_value(icat_field_name, icat_data):
     return icat_data
 
 
+class SearchAPIDatetime(datetime):
+    """
+    An alternative datetime class so datetimes can be outputted in a different format to
+    DataGateway API
+    """
+
+    @classmethod
+    def __get_validators__(cls):
+        # Use default pydantic behaviour as well as behaviour defined in this class
+        yield parse_datetime
+        yield cls.use_search_api_format
+
+    @classmethod
+    def use_search_api_format(cls, v):
+        return v.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+
+
 class PaNOSCAttribute(ABC, BaseModel):
+    _datetime_field_names = ["creationDate", "startDate", "endDate", "releaseDate"]
+
     @classmethod
     @abstractmethod
     def from_icat(cls, icat_data, required_related_fields):  # noqa: B902, N805
@@ -178,7 +198,7 @@ class Dataset(PaNOSCAttribute):
     pid: str
     title: str
     is_public: bool = Field(alias="isPublic")
-    creation_date: datetime = Field(alias="creationDate")
+    creation_date: SearchAPIDatetime = Field(alias="creationDate")
     size: Optional[int] = None
 
     documents: List["Document"] = []
@@ -218,9 +238,9 @@ class Document(PaNOSCAttribute):
     title: str
     summary: Optional[str] = None
     doi: Optional[str] = None
-    start_date: Optional[datetime] = Field(None, alias="startDate")
-    end_date: Optional[datetime] = Field(None, alias="endDate")
-    release_date: Optional[datetime] = Field(None, alias="releaseDate")
+    start_date: Optional[SearchAPIDatetime] = Field(None, alias="startDate")
+    end_date: Optional[SearchAPIDatetime] = Field(None, alias="endDate")
+    release_date: Optional[SearchAPIDatetime] = Field(None, alias="releaseDate")
     license_: Optional[str] = Field(None, alias="license")
     keywords: Optional[List[str]] = []
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datagateway-api"
-version = "4.2.0"
+version = "4.3.0"
 description = "ICAT API to interface with the DataGateway"
 license = "Apache-2.0"
 readme = "README.md"

test/search_api/endpoints/test_get_dataset_files.py

@@ -103,7 +103,7 @@ class TestSearchAPIGetDatasetFilesEndpoint:
                             "pid": "0-8401-1070-7",
                             "title": "DATASET 2",
                             "isPublic": True,
-                            "creationDate": "2013-04-01T10:56:52+00:00",
+                            "creationDate": "2013-04-01T10:56:52.000Z",
                             "size": None,
                             "documents": [],
                             "techniques": [],

test/search_api/endpoints/test_get_entity_by_pid.py

@@ -15,7 +15,7 @@ class TestSearchAPIGetByPIDEndpoint:
                     "pid": "0-8401-1070-7",
                     "title": "DATASET 2",
                     "isPublic": True,
-                    "creationDate": "2013-04-01T10:56:52+00:00",
+                    "creationDate": "2013-04-01T10:56:52.000Z",
                     "size": None,
                     "documents": [],
                     "techniques": [],
@@ -38,9 +38,9 @@ class TestSearchAPIGetByPIDEndpoint:
                     "summary": "Season identify professor happen third. Beat"
                     " professional blue clear style have. Light final summer.",
                     "doi": "0-449-78690-0",
-                    "startDate": "2000-04-03T00:00:00+00:00",
-                    "endDate": "2000-07-09T00:00:00+00:00",
-                    "releaseDate": "2000-07-05T00:00:00+00:00",
+                    "startDate": "2000-04-03T00:00:00.000Z",
+                    "endDate": "2000-07-09T00:00:00.000Z",
+                    "releaseDate": "2000-07-05T00:00:00.000Z",
                     "license": None,
                     "keywords": [
                         "read123",
@@ -148,7 +148,7 @@ class TestSearchAPIGetByPIDEndpoint:
                     "pid": "0-8401-1070-7",
                     "title": "DATASET 2",
                     "isPublic": True,
-                    "creationDate": "2013-04-01T10:56:52+00:00",
+                    "creationDate": "2013-04-01T10:56:52.000Z",
                     "size": None,
                     "documents": [
                         {
@@ -161,9 +161,9 @@ class TestSearchAPIGetByPIDEndpoint:
                             "\nPurpose study usually gas think. Machine world doctor"
                             " rise be college treat.",
                             "doi": "0-9729806-3-6",
-                            "startDate": "2000-06-04T00:00:00+00:00",
-                            "endDate": "2000-09-14T00:00:00+00:00",
-                            "releaseDate": "2000-02-10T00:00:00+00:00",
+                            "startDate": "2000-06-04T00:00:00.000Z",
+                            "endDate": "2000-09-14T00:00:00.000Z",
+                            "releaseDate": "2000-02-10T00:00:00.000Z",
                             "license": None,
                             "keywords": [
                                 "later868",
@@ -262,9 +262,9 @@ class TestSearchAPIGetByPIDEndpoint:
                     "summary": "Season identify professor happen third. Beat"
                     " professional blue clear style have. Light final summer.",
                     "doi": "0-449-78690-0",
-                    "startDate": "2000-04-03T00:00:00+00:00",
-                    "endDate": "2000-07-09T00:00:00+00:00",
-                    "releaseDate": "2000-07-05T00:00:00+00:00",
+                    "startDate": "2000-04-03T00:00:00.000Z",
+                    "endDate": "2000-07-09T00:00:00.000Z",
+                    "releaseDate": "2000-07-05T00:00:00.000Z",
                     "license": None,
                     "keywords": [
                         "read123",
@@ -351,7 +351,7 @@ class TestSearchAPIGetByPIDEndpoint:
                             "pid": "0-449-78690-0",
                             "title": "DATASET 1",
                             "isPublic": True,
-                            "creationDate": "2002-11-27T06:20:36+00:00",
+                            "creationDate": "2002-11-27T06:20:36.000Z",
                             "size": None,
                             "documents": [],
                             "techniques": [],
@@ -364,7 +364,7 @@ class TestSearchAPIGetByPIDEndpoint:
                             "pid": "0-353-84629-5",
                             "title": "DATASET 241",
                             "isPublic": True,
-                            "creationDate": "2006-11-21T17:10:42+00:00",
+                            "creationDate": "2006-11-21T17:10:42.000Z",
                             "size": None,
                             "documents": [],
                             "techniques": [],
@@ -414,7 +414,7 @@ class TestSearchAPIGetByPIDEndpoint:
                     "pid": "0-8401-1070-7",
                     "title": "DATASET 2",
                     "isPublic": True,
-                    "creationDate": "2013-04-01T10:56:52+00:00",
+                    "creationDate": "2013-04-01T10:56:52.000Z",
                     "size": None,
                     "documents": [
                         {
@@ -427,9 +427,9 @@ class TestSearchAPIGetByPIDEndpoint:
                             "\nPurpose study usually gas think. Machine world doctor "
                             "rise be college treat.",
                             "doi": "0-9729806-3-6",
-                            "startDate": "2000-06-04T00:00:00+00:00",
-                            "endDate": "2000-09-14T00:00:00+00:00",
-                            "releaseDate": "2000-02-10T00:00:00+00:00",
+                            "startDate": "2000-06-04T00:00:00.000Z",
+                            "endDate": "2000-09-14T00:00:00.000Z",
+                            "releaseDate": "2000-02-10T00:00:00.000Z",
                             "license": None,
                             "keywords": [
                                 "later868",
@@ -943,9 +943,9 @@ class TestSearchAPIGetByPIDEndpoint:
                     "summary": "Season identify professor happen third. Beat"
                     " professional blue clear style have. Light final summer.",
                     "doi": "0-449-78690-0",
-                    "startDate": "2000-04-03T00:00:00+00:00",
-                    "endDate": "2000-07-09T00:00:00+00:00",
-                    "releaseDate": "2000-07-05T00:00:00+00:00",
+                    "startDate": "2000-04-03T00:00:00.000Z",
+                    "endDate": "2000-07-09T00:00:00.000Z",
+                    "releaseDate": "2000-07-05T00:00:00.000Z",
                     "license": None,
                     "keywords": [
                         "read123",
@@ -1032,7 +1032,7 @@ class TestSearchAPIGetByPIDEndpoint:
                             "pid": "0-449-78690-0",
                             "title": "DATASET 1",
                             "isPublic": True,
-                            "creationDate": "2002-11-27T06:20:36+00:00",
+                            "creationDate": "2002-11-27T06:20:36.000Z",
                             "size": None,
                             "documents": [],
                             "techniques": [],
@@ -1045,7 +1045,7 @@ class TestSearchAPIGetByPIDEndpoint:
                             "pid": "0-353-84629-5",
                             "title": "DATASET 241",
                             "isPublic": True,
-                            "creationDate": "2006-11-21T17:10:42+00:00",
+                            "creationDate": "2006-11-21T17:10:42.000Z",
                             "size": None,
                             "documents": [],
                             "techniques": [],