Merge pull request #302 from ral-facilities/feature/search-endpoints-…

…#268

Implement Search API Endpoints

.github/workflows/ci-build.yml

@@ -59,6 +59,17 @@ jobs:
         run: |
           ansible-playbook icat-ansible/icatdb-minimal-hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
 
+      # rootUserNames needs editing as anon/anon is used in search API and required to pass endpoint tests
+      - name: Add anon user to rootUserNames
+        run: |
+          awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
+      - name: Apply rootUserNames change
+        run: |
+          mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
+      - name: Reinstall ICAT Server
+        run: |
+          cd /home/runner/install/icat.server/ && ./setup -vv install
+
       - name: Checkout DataGateway API
         uses: actions/checkout@v2
 

datagateway_api/src/api_start_utils.py

@@ -233,52 +233,44 @@ def create_api_endpoints(flask_app, api, spec):
         }
 
         for endpoint_name, entity_name in search_api_entity_endpoints.items():
-            get_search_endpoint_resource = get_search_endpoint(
-                endpoint_name, entity_name,
-            )
+            get_search_endpoint_resource = get_search_endpoint(entity_name)
             api.add_resource(
                 get_search_endpoint_resource,
                 f"{search_api_extension}/{endpoint_name}",
                 endpoint=f"search_api_get_{endpoint_name}",
             )
             spec.path(resource=get_search_endpoint_resource, api=api)
 
-            get_single_endpoint_resource = get_single_endpoint(
-                endpoint_name, entity_name,
-            )
+            get_single_endpoint_resource = get_single_endpoint(entity_name)
             api.add_resource(
                 get_single_endpoint_resource,
-                f"{search_api_extension}/{endpoint_name}/<int:pid>",
+                f"{search_api_extension}/{endpoint_name}/<string:pid>",
                 endpoint=f"search_api_get_single_{endpoint_name}",
             )
             spec.path(resource=get_single_endpoint_resource, api=api)
 
-            get_number_count_endpoint_resource = get_number_count_endpoint(
-                endpoint_name, entity_name,
-            )
+            get_number_count_endpoint_resource = get_number_count_endpoint(entity_name)
             api.add_resource(
                 get_number_count_endpoint_resource,
                 f"{search_api_extension}/{endpoint_name}/count",
                 endpoint=f"search_api_count_{endpoint_name}",
             )
             spec.path(resource=get_number_count_endpoint_resource, api=api)
 
-        get_files_endpoint_resource = get_files_endpoint(
-            search_api_entity_endpoints["datasets"], "datasets",
-        )
+        get_files_endpoint_resource = get_files_endpoint("File")
         api.add_resource(
             get_files_endpoint_resource,
-            f"{search_api_extension}/datasets/<int:pid>/files",
+            f"{search_api_extension}/datasets/<string:pid>/files",
             endpoint="search_api_get_dataset_files",
         )
         spec.path(resource=get_files_endpoint_resource, api=api)
 
         get_number_count_files_endpoint_resource = get_number_count_files_endpoint(
-            search_api_entity_endpoints["datasets"], "datasets",
+            "File",
         )
         api.add_resource(
             get_number_count_files_endpoint_resource,
-            f"{search_api_extension}/datasets/<int:pid>/files/count",
+            f"{search_api_extension}/datasets/<string:pid>/files/count",
             endpoint="search_api_count_dataset_files",
         )
         spec.path(resource=get_number_count_files_endpoint_resource, api=api)

datagateway_api/src/common/filter_order_handler.py

@@ -8,6 +8,8 @@
 )
 from datagateway_api.src.search_api.filters import SearchAPIIncludeFilter
 from datagateway_api.src.search_api.panosc_mappings import mappings
+from datagateway_api.src.search_api.query import SearchAPIQuery
+
 
 log = logging.getLogger()
 
@@ -45,6 +47,13 @@ def apply_filters(self, query):
         self.sort_filters()
 
         for query_filter in self.filters:
+            # Using `type()` because we only want the Python ICAT version, don't want
+            # the code to catch objects that inherit from the class e.g.
+            # `SearchAPIIncludeFilter`
+            if type(query_filter) is PythonICATIncludeFilter and isinstance(
+                query, SearchAPIQuery,
+            ):
+                query = query.icat_query.query
             query_filter.apply_filter(query)
 
     def add_icat_relations_for_non_related_fields_of_panosc_related_entities(
@@ -99,6 +108,26 @@ def add_icat_relations_for_non_related_fields_of_panosc_related_entities(
                 python_icat_include_filter = PythonICATIncludeFilter(icat_relations)
                 self.filters.append(python_icat_include_filter)
 
+    def add_icat_relations_for_panosc_non_related_fields(
+        self, panosc_entity_name,
+    ):
+        """
+        Retrieve ICAT relations and create a `PythonICATIncludeFilter` for these ICAT
+        relations
+
+        :param panosc_entity_name: A PaNOSC entity name e.g. "Dataset"
+        :type panosc_entity_name: :class:`str`
+        """
+
+        icat_relations = mappings.get_icat_relations_for_panosc_non_related_fields(
+            panosc_entity_name,
+        )
+
+        # Remove any duplicate ICAT relations
+        icat_relations = list(dict.fromkeys(icat_relations))
+        if icat_relations:
+            self.filters.append(PythonICATIncludeFilter(icat_relations))
+
     def merge_python_icat_limit_skip_filters(self):
         """
         When there are both limit and skip filters in a request, merge them into the

datagateway_api/src/resources/entities/entity_endpoint.py

@@ -9,9 +9,9 @@
 
 def get_endpoint(name, entity_type, backend, **kwargs):
     """
-    Given an entity name generate a flask_restful Resource class.
-    In main.py these generated classes are registered with the api e.g
-    api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")
+    Given an entity name, generate a flask_restful `Resource` class. In
+    `create_api_endpoints()`, these generated classes are registered with the API e.g.
+    `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles")
 
     :param name: The name of the entity
     :type name: :class:`str`
@@ -168,9 +168,9 @@ def patch(self):
 
 def get_id_endpoint(name, entity_type, backend, **kwargs):
     """
-    Given an entity name generate a flask_restful Resource class.
-    In main.py these generated classes are registered with the api e.g
-    api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/<int:id_>")
+    Given an entity name, generate a flask_restful `Resource` class. In
+    `create_api_endpoints()`, these generated classes are registered with the API e.g.
+    `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/<int:id_>")`
 
     :param name: The name of the entity
     :type name: :class:`str`
@@ -306,9 +306,9 @@ def patch(self, id_):
 
 def get_count_endpoint(name, entity_type, backend, **kwargs):
     """
-    Given an entity name generate a flask_restful Resource class.
-    In main.py these generated classes are registered with the api e.g
-    api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/count")
+    Given an entity name, generate a flask_restful `Resource` class. In
+    `create_api_endpoints()`, these generated classes are registered with the API e.g.
+    `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/count")`
 
     :param name: The name of the entity
     :type name: :class:`str`
@@ -363,9 +363,9 @@ def get(self):
 
 def get_find_one_endpoint(name, entity_type, backend, **kwargs):
     """
-    Given an entity name generate a flask_restful Resource class.
-    In main.py these generated classes are registered with the api e.g
-    api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/findone")
+    Given an entity name, generate a flask_restful `Resource` class. In
+    `create_api_endpoints()`, these generated classes are registered with the API e.g.
+    `api.add_resource(get_endpoint("Datafiles", DATAFILE), "/datafiles/findone")`
 
     :param name: The name of the entity
     :type name: :class:`str`

datagateway_api/src/resources/search_api_endpoints.py

@@ -8,92 +8,123 @@
     get_files,
     get_files_count,
     get_search,
-    get_with_id,
+    get_with_pid,
 )
 
 log = logging.getLogger()
 
 
-def get_search_endpoint(endpoint_name, entity_name):
+def get_search_endpoint(entity_name):
     """
-    TODO - Add docstring
+    Given an entity name, generate a flask_restful `Resource` class. In
+    `create_api_endpoints()`, these generated classes are registered with the API e.g.
+    `api.add_resource(get_search_endpoint("Dataset"), "/datasets")`
+
+    :param entity_name: Name of the entity
+    :type entity_name: :class:`str`
+    :return: Generated endpoint class
     """
 
     class Endpoint(Resource):
         def get(self):
             filters = get_filters_from_query_string("search_api", entity_name)
             log.debug("Filters: %s", filters)
-            return get_search(endpoint_name, entity_name, filters), 200
+            return get_search(entity_name, filters), 200
 
         # TODO - Add `get.__doc__`
 
     Endpoint.__name__ = entity_name
     return Endpoint
 
 
-def get_single_endpoint(endpoint_name, entity_name):
+def get_single_endpoint(entity_name):
     """
-    TODO - Add docstring
+    Given an entity name, generate a flask_restful `Resource` class. In
+    `create_api_endpoints()`, these generated classes are registered with the API e.g.
+    `api.add_resource(get_single_endpoint("Dataset"), "/datasets/<string:pid>")`
+
+    :param entity_name: Name of the entity
+    :type entity_name: :class:`str`
+    :return: Generated endpoint class
     """
 
     class EndpointWithID(Resource):
         def get(self, pid):
             filters = get_filters_from_query_string("search_api", entity_name)
             log.debug("Filters: %s", filters)
-            return get_with_id(entity_name, pid), 200
+            return get_with_pid(entity_name, pid, filters), 200
 
         # TODO - Add `get.__doc__`
 
     EndpointWithID.__name__ = entity_name
     return EndpointWithID
 
 
-def get_number_count_endpoint(endpoint_name, entity_name):
+def get_number_count_endpoint(entity_name):
     """
-    TODO - Add docstring
+    Given an entity name, generate a flask_restful `Resource` class. In
+    `create_api_endpoints()`, these generated classes are registered with the API e.g.
+    `api.add_resource(get_number_count_endpoint("Dataset"), "/datasets/count")`
+
+    :param entity_name: Name of the entity
+    :type entity_name: :class:`str`
+    :return: Generated endpoint class
     """
 
     class CountEndpoint(Resource):
         def get(self):
             # Only WHERE included on count endpoints
             filters = get_filters_from_query_string("search_api", entity_name)
             log.debug("Filters: %s", filters)
-            return get_count(entity_name), 200
+            return get_count(entity_name, filters), 200
 
         # TODO - Add `get.__doc__`
 
     CountEndpoint.__name__ = entity_name
     return CountEndpoint
 
 
-def get_files_endpoint(endpoint_name, entity_name):
+def get_files_endpoint(entity_name):
     """
-    TODO - Add docstring
+    Given an entity name, generate a flask_restful `Resource` class. In
+    `create_api_endpoints()`, these generated classes are registered with the API e.g.
+    `api.add_resource(get_files_endpoint("Dataset"), "/datasets/<string:pid>/files")`
+
+    :param entity_name: Name of the entity
+    :type entity_name: :class:`str`
+    :return: Generated endpoint class
     """
 
     class FilesEndpoint(Resource):
         def get(self, pid):
             filters = get_filters_from_query_string("search_api", entity_name)
             log.debug("Filters: %s", filters)
-            return get_files(entity_name), 200
+            return get_files(entity_name, pid, filters), 200
 
         # TODO - Add `get.__doc__`
 
     FilesEndpoint.__name__ = entity_name
     return FilesEndpoint
 
 
-def get_number_count_files_endpoint(endpoint_name, entity_name):
+def get_number_count_files_endpoint(entity_name):
     """
-    TODO - Add docstring
+    Given an entity name, generate a flask_restful `Resource` class. In
+    `create_api_endpoints()`, these generated classes are registered with the API e.g.
+    `api.add_resource(get_number_count_files_endpoint("Dataset"),
+    "/datasets<string:pid>/files/count")`
+
+    :param entity_name: Name of the entity
+    :type entity_name: :class:`str`
+    :return: Generated endpoint class
     """
 
     class CountFilesEndpoint(Resource):
         def get(self, pid):
             # Only WHERE included on count endpoints
             filters = get_filters_from_query_string("search_api", entity_name)
             log.debug("Filters: %s", filters)
-            return get_files_count(entity_name, pid)
+            return get_files_count(entity_name, filters, pid)
 
         # TODO - Add `get.__doc__`