Merge pull request #304 from ral-facilities/include-icat-relations-fo…

…r-non-related-panosc-fields

Include ICAT relations for non-related PaNOSC fields

datagateway_api/src/datagateway_api/filter_order_handler.py → datagateway_api/src/common/filter_order_handler.py

@@ -1,10 +1,13 @@
 import logging
 
 from datagateway_api.src.datagateway_api.icat.filters import (
+    PythonICATIncludeFilter,
     PythonICATLimitFilter,
     PythonICATOrderFilter,
     PythonICATSkipFilter,
 )
+from datagateway_api.src.search_api.filters import SearchAPIIncludeFilter
+from datagateway_api.src.search_api.panosc_mappings import mappings
 
 log = logging.getLogger()
 
@@ -44,6 +47,58 @@ def apply_filters(self, query):
         for query_filter in self.filters:
             query_filter.apply_filter(query)
 
+    def add_icat_relations_for_non_related_fields_of_panosc_related_entities(
+        self, panosc_entity_name,
+    ):
+        """
+        When there are Search API included filters, get the ICAT relations (if any) for
+        the non-related fields of all the entities in the relations. Once retrieved,
+        add them to the `included_filters` list of a `PythonICATIncludeFilter` object
+        that may already exist in `self.filters`. If such filter does not exist in
+        `self.filters` then create a new `PythonICATIncludeFilter` object, passing the
+        ICAT relations to it. Doing this will ensure that ICAT related entities that
+        map to non-related PaNOSC fields are included in the call made to ICAT.
+
+        A `PythonICATIncludeFilter` object can exist in `self.filters` when one is
+        created and added in the `get_search` method. This is done when the the PaNOSC
+        entity for which search is been retrieved has non-related fields that have
+        ICAT relations. For example, the Document entity has non-related fields that
+        map to the `keywords` and `type` ICAT entities that are related to the
+        `investigation` entity.
+
+        :param panosc_entity_name: A PaNOSC entity name e.g. "Dataset"
+        :type panosc_entity_name: :class:`str`
+        """
+
+        python_icat_include_filter = None
+        icat_relations = []
+        for filter_ in self.filters:
+            if type(filter_) == PythonICATIncludeFilter:
+                # Using `type` as `isinstance` would return `True` for any class that
+                # inherits `PythonICATIncludeFilter` e.g. `SearchAPIIncludeFilter`.`
+                python_icat_include_filter = filter_
+            elif isinstance(filter_, SearchAPIIncludeFilter):
+                included_filters = filter_.included_filters
+                for included_filter in included_filters:
+                    icat_relations.extend(
+                        mappings.get_icat_relations_for_non_related_fields_of_panosc_relation(  # noqa: B950
+                            panosc_entity_name, included_filter,
+                        ),
+                    )
+
+        if icat_relations:
+            log.info(
+                "Including ICAT relations of non-related fields of related PaNOSC "
+                "entities",
+            )
+            # Remove any duplicate ICAT relations
+            icat_relations = list(dict.fromkeys(icat_relations))
+            if python_icat_include_filter:
+                python_icat_include_filter.included_filters.extend(icat_relations)
+            else:
+                python_icat_include_filter = PythonICATIncludeFilter(icat_relations)
+                self.filters.append(python_icat_include_filter)
+
     def merge_python_icat_limit_skip_filters(self):
         """
         When there are both limit and skip filters in a request, merge them into the

datagateway_api/src/datagateway_api/database/helpers.py

@@ -11,6 +11,7 @@
     BadRequestError,
     MissingRecordError,
 )
+from datagateway_api.src.common.filter_order_handler import FilterOrderHandler
 from datagateway_api.src.common.helpers import map_distinct_attributes_to_results
 from datagateway_api.src.datagateway_api.database.filters import (
     DatabaseDistinctFieldFilter,
@@ -25,7 +26,6 @@
     INVESTIGATIONINSTRUMENT,
     SESSION,
 )
-from datagateway_api.src.datagateway_api.filter_order_handler import FilterOrderHandler
 
 
 log = logging.getLogger()

datagateway_api/src/datagateway_api/icat/helpers.py

@@ -21,7 +21,7 @@
     MissingRecordError,
     PythonICATError,
 )
-from datagateway_api.src.datagateway_api.filter_order_handler import FilterOrderHandler
+from datagateway_api.src.common.filter_order_handler import FilterOrderHandler
 from datagateway_api.src.datagateway_api.icat.filters import (
     PythonICATLimitFilter,
     PythonICATWhereFilter,

datagateway_api/src/search_api/helpers.py

@@ -1,6 +1,8 @@
 import logging
 
-from datagateway_api.src.datagateway_api.filter_order_handler import FilterOrderHandler
+from datagateway_api.src.common.filter_order_handler import FilterOrderHandler
+from datagateway_api.src.datagateway_api.icat.filters import PythonICATIncludeFilter
+from datagateway_api.src.search_api.panosc_mappings import mappings
 from datagateway_api.src.search_api.query import SearchAPIQuery
 from datagateway_api.src.search_api.session_handler import (
     client_manager,
@@ -15,6 +17,14 @@
 def get_search(endpoint_name, entity_name, filters):
     log.debug("Entity Name: %s, Filters: %s", entity_name, filters)
 
+    icat_relations = mappings.get_icat_relations_for_panosc_non_related_fields(
+        entity_name,
+    )
+    # Remove any duplicate ICAT relations
+    icat_relations = list(dict.fromkeys(icat_relations))
+    if icat_relations:
+        filters.append(PythonICATIncludeFilter(icat_relations))
+
     query = SearchAPIQuery(entity_name)
 
     filter_handler = FilterOrderHandler()

datagateway_api/src/search_api/panosc_mappings.py

@@ -98,5 +98,108 @@ def get_panosc_related_entity_name(
 
         return panosc_related_entity_name
 
+    def get_panosc_non_related_field_names(self, panosc_entity_name):
+        """
+        This function retrieves the names of the non related fields of a given PaNOSC
+        entity.
+
+        :param panosc_entity_name: A PaNOSC entity name e.g. "Dataset"
+        :type panosc_entity_name: :class:`str`
+        :return: List containing the names of the non related fields of the given
+            PaNOSC entity
+        :raises FilterError: If mappings for the given entity name cannot be found
+        """
+        try:
+            entity_mappings = self.mappings[panosc_entity_name]
+        except KeyError:
+            raise FilterError(
+                f"Cannot find mappings for {[panosc_entity_name]} PaNOSC entity",
+            )
+
+        non_related_field_names = []
+        for mapping_key, mapping_value in entity_mappings.items():
+            # The mappings for the non-related fields are of type `str` and sometimes
+            # `list' whereas for the related fields, they are of type `dict`.
+            if mapping_key != "base_icat_entity" and (
+                isinstance(mapping_value, str) or isinstance(mapping_value, list)
+            ):
+                non_related_field_names.append(mapping_key)
+
+        return non_related_field_names
+
+    def get_icat_relations_for_panosc_non_related_fields(self, panosc_entity_name):
+        """
+        This function retrieves the ICAT relations for the non related fields of a
+        given PaNOSC entity.
+
+        :param panosc_entity_name: A PaNOSC entity name e.g. "Dataset"
+        :type panosc_entity_name: :class:`str`
+        :return: List containing the ICAT relations for the non related fields of the
+            given PaNOSC entity
+        """
+        icat_relations = []
+
+        field_names = self.get_panosc_non_related_field_names(panosc_entity_name)
+        for field_name in field_names:
+            _, icat_mapping = self.get_icat_mapping(panosc_entity_name, field_name)
+
+            if not isinstance(icat_mapping, list):
+                icat_mapping = [icat_mapping]
+
+            for mapping in icat_mapping:
+                split_mapping = mapping.split(".")
+                if len(split_mapping) > 1:
+                    # Remove the last split element because it is an ICAT
+                    # field name and is not therefore part of the relation
+                    split_mapping = split_mapping[:-1]
+                    split_mapping = ".".join(split_mapping)
+                    icat_relations.append(split_mapping)
+
+        return icat_relations
+
+    def get_icat_relations_for_non_related_fields_of_panosc_relation(
+        self, panosc_entity_name, entity_relation,
+    ):
+        """
+        THis function retrieves the ICAT relations for the non related fields of all the
+        PaNOSC entities that form a given PaNOSC entity relation which is applied to a
+        given PaNOSC entity. Relations can be non-nested or nested. Those that are
+        nested are represented in a dotted format e.g. "documents.members.person". When
+        a given relation is nested, this function retrieves the ICAT relations for the
+        first PaNOSC entity and then recursively calls itself until the ICAT relations
+        for the last PaNOSC entity in the relation are retrieved.
+
+        :param panosc_entity_name: A PaNOSC entity name e.g. "Dataset" to which the
+            PaNOSC entity relation is applied
+        :type panosc_entity_name: :class:`str`
+        :param panosc_entity_name: A PaNOSC entity relation e.g. "documents" or
+            "documents.members.person" if nested
+        :type panosc_entity_name: :class:`str`
+        :return: List containing the ICAT relations for the non related fields of all
+            the PaNOSC entitities that form the given PaNOSC entity relation
+        """
+        icat_relations = []
+
+        split_entity_relation = entity_relation.split(".")
+        related_entity_name, icat_field_name = self.get_icat_mapping(
+            panosc_entity_name, split_entity_relation[0],
+        )
+        relations = self.get_icat_relations_for_panosc_non_related_fields(
+            related_entity_name,
+        )
+        icat_relations.extend(relations)
+
+        if len(split_entity_relation) > 1:
+            entity_relation = ".".join(split_entity_relation[1:])
+            relations = self.get_icat_relations_for_non_related_fields_of_panosc_relation(  # noqa: B950
+                related_entity_name, entity_relation,
+            )
+            icat_relations.extend(relations)
+
+        for i, icat_relation in enumerate(icat_relations):
+            icat_relations[i] = f"{icat_field_name}.{icat_relation}"
+
+        return icat_relations
+
 
 mappings = PaNOSCMappings()

test/conftest.py

@@ -3,20 +3,39 @@
 from unittest.mock import mock_open, patch
 
 from flask import Flask
+from icat.client import Client
+from icat.query import Query
 import pytest
 
 from datagateway_api.src.api_start_utils import (
     create_api_endpoints,
     create_app_infrastructure,
 )
-from datagateway_api.src.common.config import APIConfig
+from datagateway_api.src.common.config import APIConfig, Config
 from datagateway_api.src.datagateway_api.database.helpers import (
     delete_row_by_id,
     insert_row_into_table,
 )
 from datagateway_api.src.datagateway_api.database.models import SESSION
 
 
+@pytest.fixture(scope="package")
+def icat_client():
+    client = Client(
+        Config.config.datagateway_api.icat_url,
+        checkCert=Config.config.datagateway_api.icat_check_cert,
+    )
+    client.login(
+        Config.config.test_mechanism, Config.config.test_user_credentials.dict(),
+    )
+    return client
+
+
+@pytest.fixture()
+def icat_query(icat_client):
+    return Query(icat_client, "Investigation")
+
+
 @pytest.fixture()
 def bad_credentials_header():
     return {"Authorization": "Bearer Invalid"}

test/datagateway_api/icat/conftest.py

@@ -3,9 +3,7 @@
 
 from dateutil.tz import tzlocal
 from flask import Flask
-from icat.client import Client
 from icat.exception import ICATNoObjectError
-from icat.query import Query
 import pytest
 
 from datagateway_api.src.api_start_utils import (
@@ -17,28 +15,11 @@
 from test.datagateway_api.icat.test_query import prepare_icat_data_for_assertion
 
 
-@pytest.fixture(scope="package")
-def icat_client():
-    client = Client(
-        Config.config.datagateway_api.icat_url,
-        checkCert=Config.config.datagateway_api.icat_check_cert,
-    )
-    client.login(
-        Config.config.test_mechanism, Config.config.test_user_credentials.dict(),
-    )
-    return client
-
-
 @pytest.fixture()
 def valid_icat_credentials_header(icat_client):
     return {"Authorization": f"Bearer {icat_client.sessionId}"}
 
 
-@pytest.fixture()
-def icat_query(icat_client):
-    return Query(icat_client, "Investigation")
-
-
 def create_investigation_test_data(client, num_entities=1):
     test_data = []
 

test/datagateway_api/icat/filters/test_limit_filter.py

@@ -3,7 +3,7 @@
 import pytest
 
 from datagateway_api.src.common.exceptions import FilterError
-from datagateway_api.src.datagateway_api.filter_order_handler import FilterOrderHandler
+from datagateway_api.src.common.filter_order_handler import FilterOrderHandler
 from datagateway_api.src.datagateway_api.icat.filters import (
     icat_set_limit,
     PythonICATLimitFilter,

test/datagateway_api/icat/filters/test_order_filter.py

@@ -2,7 +2,7 @@
 from typing_extensions import OrderedDict
 
 from datagateway_api.src.common.exceptions import FilterError
-from datagateway_api.src.datagateway_api.filter_order_handler import FilterOrderHandler
+from datagateway_api.src.common.filter_order_handler import FilterOrderHandler
 from datagateway_api.src.datagateway_api.icat.filters import PythonICATOrderFilter
 
 

test/datagateway_api/icat/filters/test_where_filter.py

@@ -1,7 +1,7 @@
 import pytest
 
 from datagateway_api.src.common.exceptions import BadRequestError, FilterError
-from datagateway_api.src.datagateway_api.filter_order_handler import FilterOrderHandler
+from datagateway_api.src.common.filter_order_handler import FilterOrderHandler
 from datagateway_api.src.datagateway_api.icat.filters import PythonICATWhereFilter