diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 82ca1b8d..1d9a4a47 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -11,146 +11,153 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8', '3.9'] + python-version: ["3.6", "3.7", "3.8", "3.9"] name: Python ${{ matrix.python-version }} Build & Tests steps: - - name: Add apt repo - run: sudo add-apt-repository universe - - # Setup Java & Python - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: 8 - java-package: jdk - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - architecture: x64 - - # ICAT Ansible clone and install dependencies - - name: Checkout icat-ansible - uses: actions/checkout@v2 - with: - repository: icatproject-contrib/icat-ansible - ref: master - path: icat-ansible - - name: Install Ansible - run: pip install -r icat-ansible/requirements.txt - - # Prep for running the playbook - - name: Create hosts file - run: echo -e "[icatdb-minimal-hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts - - name: Prepare vault pass - run: echo -e "icattravispw" > icat-ansible/vault_pass.txt - - name: Move vault to directory it'll get detected by Ansible - run: mv icat-ansible/vault.yml icat-ansible/group_vars/all - - name: Replace default payara user with Actions user - run: | - sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml - - # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions - - name: Change hostname to localhost - run: sudo hostname -b localhost - - # Create local instance of ICAT - - name: Run ICAT Ansible Playbook - run: | - ansible-playbook icat-ansible/icatdb-minimal-hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv - - - name: Checkout DataGateway API - uses: actions/checkout@v2 - - # Prep for using the API for tests - - name: Create log file - run: touch logs.log - - name: Configure log file location - run: echo "`jq -r --arg REPO_DIR "$GITHUB_WORKSPACE/logs.log" \ - '.log_location=$REPO_DIR' datagateway_api/config.json.example`" > datagateway_api/config.json.example - - name: Create config.json - run: cp datagateway_api/config.json.example datagateway_api/config.json - - name: Create search_api_mapping.json - run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json - - # Install Nox, Poetry and API's dependencies - - name: Install Nox - run: pip install nox==2020.8.22 - - name: Install Poetry - run: pip install poetry==1.1.9 - - name: Install dependencies - run: poetry install - - - name: Add dummy data to icatdb - run: | - poetry run python -m util.icat_db_generator - - # Run Nox tests session, saves and uploads a coverage report to codecov - - name: Run Nox tests session - run: nox -p ${{ matrix.python-version }} -s tests -- --cov=datagateway_api --cov-report=xml - - name: Upload code coverage report - if: matrix.python-version == '3.6' - uses: codecov/codecov-action@v1 + - name: Add apt repo + run: sudo add-apt-repository universe + + # Setup Java & Python + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: 8 + java-package: jdk + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + + # ICAT Ansible clone and install dependencies + - name: Checkout icat-ansible + uses: actions/checkout@v2 + with: + repository: icatproject-contrib/icat-ansible + ref: master + path: icat-ansible + - name: Install Ansible + run: pip install -r icat-ansible/requirements.txt + + # Prep for running the playbook + - name: Create hosts file + run: echo -e "[icatdb-minimal-hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts + - name: Prepare vault pass + run: echo -e "icattravispw" > icat-ansible/vault_pass.txt + - name: Move vault to directory it'll get detected by Ansible + run: mv icat-ansible/vault.yml icat-ansible/group_vars/all + - name: Replace default payara user with Actions user + run: | + sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml + + # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions + - name: Change hostname to localhost + run: sudo hostname -b localhost + + # Create local instance of ICAT + - name: Run ICAT Ansible Playbook + run: | + ansible-playbook icat-ansible/icatdb-minimal-hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv + + - name: Checkout DataGateway API + uses: actions/checkout@v2 + + # Prep for using the API for tests + - name: Create log file + run: touch logs.log + - name: Configure log file location + run: echo "`jq -r --arg REPO_DIR "$GITHUB_WORKSPACE/logs.log" \ + '.log_location=$REPO_DIR' datagateway_api/config.json.example`" > datagateway_api/config.json.example + - name: Create config.json + run: cp datagateway_api/config.json.example datagateway_api/config.json + - name: Create search_api_mapping.json + run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json + + # Install Nox, Poetry and API's dependencies + - name: Install Nox + run: pip install nox==2020.8.22 + - name: Install Poetry + run: pip install poetry==1.1.9 + + # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 + - name: Uninstall setuptools + run: poetry run pip uninstall -y setuptools + - name: Install older setuptools + run: poetry run pip install 'setuptools<58.0.0' + + - name: Install dependencies + run: poetry install + + - name: Add dummy data to icatdb + run: | + poetry run python -m util.icat_db_generator + + # Run Nox tests session, saves and uploads a coverage report to codecov + - name: Run Nox tests session + run: nox -p ${{ matrix.python-version }} -s tests -- --cov=datagateway_api --cov-report=xml + - name: Upload code coverage report + if: matrix.python-version == '3.6' + uses: codecov/codecov-action@v1 linting: runs-on: ubuntu-20.04 name: Linting steps: - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.9.7' - architecture: x64 - - name: Checkout DataGateway API - uses: actions/checkout@v2 - - - name: Install Nox - run: pip install nox==2020.8.22 - - name: Install Poetry - run: pip install poetry==1.1.9 - - - name: Run Nox lint session - run: nox -s lint + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.9.7" + architecture: x64 + - name: Checkout DataGateway API + uses: actions/checkout@v2 + + - name: Install Nox + run: pip install nox==2020.8.22 + - name: Install Poetry + run: pip install poetry==1.1.9 + + - name: Run Nox lint session + run: nox -s lint formatting: runs-on: ubuntu-20.04 name: Code Formatting steps: - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.9.7' - architecture: x64 - - name: Checkout DataGateway API - uses: actions/checkout@v2 - - - name: Install Nox - run: pip install nox==2020.8.22 - - name: Install Poetry - run: pip install poetry==1.1.9 - - - name: Run Nox black session - run: nox -s black + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.9.7" + architecture: x64 + - name: Checkout DataGateway API + uses: actions/checkout@v2 + + - name: Install Nox + run: pip install nox==2020.8.22 + - name: Install Poetry + run: pip install poetry==1.1.9 + + - name: Run Nox black session + run: nox -s black safety: runs-on: ubuntu-20.04 name: Dependency Safety steps: - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.9.7' - architecture: x64 - - name: Checkout DataGateway API - uses: actions/checkout@v2 - - - name: Install Nox - run: pip install nox==2020.8.22 - - name: Install Poetry - run: pip install poetry==1.1.9 - - - name: Run Nox safety session - run: nox -s safety + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.9.7" + architecture: x64 + - name: Checkout DataGateway API + uses: actions/checkout@v2 + + - name: Install Nox + run: pip install nox==2020.8.22 + - name: Install Poetry + run: pip install poetry==1.1.9 + + - name: Run Nox safety session + run: nox -s safety generator-script-testing: runs-on: ubuntu-20.04 @@ -169,7 +176,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: '3.9.7' + python-version: "3.9.7" architecture: x64 # ICAT Ansible clone and install dependencies @@ -211,6 +218,13 @@ jobs: run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json - name: Install Poetry run: pip install poetry==1.1.9 + + # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 + - name: Uninstall setuptools + run: poetry run pip uninstall -y setuptools + - name: Install older setuptools + run: poetry run pip install 'setuptools<58.0.0' + - name: Install dependencies run: poetry install @@ -238,7 +252,6 @@ jobs: - name: Diff SQL dumps run: diff -s ~/generator_script_dump_1.sql ~/generator_script_dump_2.sql - # Drop and re-create icatdb to remove generated data - name: Drop icatdb run: mysql -picatdbuserpw -uicatdbuser -e 'DROP DATABASE icatdb;' @@ -257,6 +270,13 @@ jobs: run: cp datagateway_api/config.json.example datagateway_api/config.json - name: Create search_api_mapping.json run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json + + # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99 + - name: Uninstall setuptools + run: poetry run pip uninstall -y setuptools + - name: Install older setuptools + run: poetry run pip install 'setuptools<58.0.0' + - name: Install dependencies run: poetry install diff --git a/CHANGELOG.md b/CHANGELOG.md index fec3063b..94d553e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,46 @@ +## v3.5.1 (2022-01-31) +### Fix +* Fix nested relations bug #261 ([`67fcbfe`](https://github.com/ral-facilities/datagateway-api/commit/67fcbfe2a35ca2b7e007a1a6d78105b2e46b0b5f)) +* Reference self instead of fixed instance #301 ([`40f5662`](https://github.com/ral-facilities/datagateway-api/commit/40f566279543871c5b7b1eab2c8b74050d8b3525)) + +## v3.5.0 (2022-01-31) +### Feature +* Implement basic version of `SearchAPIIncludeFilter` #261 ([`f2f53c9`](https://github.com/ral-facilities/datagateway-api/commit/f2f53c92229d052ae697787eb80a35dcd2ea3b45)) + +### Fix +* Fix list type field checking in Python 3.6 #265 ([`691a59e`](https://github.com/ral-facilities/datagateway-api/commit/691a59ea3f850475572c3a877fb739e5216c6fe7)) + +### Documentation +* Add new comments and fix existing #265 ([`3f1b1cf`](https://github.com/ral-facilities/datagateway-api/commit/3f1b1cffdd1e57ab4eb1227b13e0906424adefd0)) + +## v3.4.0 (2022-01-31) +### Feature +* Implement `regexp` operator #297 ([`bf3fe0e`](https://github.com/ral-facilities/datagateway-api/commit/bf3fe0ef2ac582d55dbd881edf6a81a93625ce91)) +* Implement `neq` operator #297 ([`9094bbb`](https://github.com/ral-facilities/datagateway-api/commit/9094bbb894ead20a53fadfd0e24b264af29548b9)) +* Implement `nin` operator #297 ([`00dbba5`](https://github.com/ral-facilities/datagateway-api/commit/00dbba525d5cd86cb5577f3b1621a7042cdd2fa0)) +* Implement `inq` operator #297 ([`fc1cf19`](https://github.com/ral-facilities/datagateway-api/commit/fc1cf194454a4da60652b1f68df278c4624ddc11)) +* Implement `between` operator #297 ([`4736888`](https://github.com/ral-facilities/datagateway-api/commit/4736888bf76cda0dbc00f997443ed565f0f5e760)) + +## v3.3.0 (2022-01-31) +### Feature +* Add function to get PaNOSC to ICAT mapping for where filter #260 ([`34b1d81`](https://github.com/ral-facilities/datagateway-api/commit/34b1d819482aa3efdb4f8da321125d3e40d76617)) +* Convert PaNOSC to ICAT for where filter fields #260 ([`ff9595d`](https://github.com/ral-facilities/datagateway-api/commit/ff9595d2f571211db79dea02f702d4148b8879f3)) + +### Fix +* Fix example mapping file ([`3802cc9`](https://github.com/ral-facilities/datagateway-api/commit/3802cc9ee71a355d0ad87529f65112e8c3f8b881)) +* Update `__str__()` for WHERE filter to cope with applying filter #260 ([`8d259d7`](https://github.com/ral-facilities/datagateway-api/commit/8d259d75a28414f26cc569293720ef4e306e6844)) + +### Documentation +* Add docstring to static function #260 ([`618f6b9`](https://github.com/ral-facilities/datagateway-api/commit/618f6b9fead88f61a346b90cb2b85a90877b0410)) + +## v3.2.0 (2022-01-31) +### Feature +* Add class to represent nested conditions #259 ([`583cbf2`](https://github.com/ral-facilities/datagateway-api/commit/583cbf29744b72c020429b61ae7442b19acef231)) +* Add first pass of query param implementation #259 ([`ee668e3`](https://github.com/ral-facilities/datagateway-api/commit/ee668e38cd43354851163616a93924ad84e14b90)) + ## v3.1.1 (2021-12-15) ### Fix * Correct reference to class name #264 ([`fc4c180`](https://github.com/ral-facilities/datagateway-api/commit/fc4c18085ab496d838e8d1e9e3f8c77f07826e9d)) diff --git a/datagateway_api/src/common/filters.py b/datagateway_api/src/common/filters.py index c99e26eb..bd0e636c 100644 --- a/datagateway_api/src/common/filters.py +++ b/datagateway_api/src/common/filters.py @@ -27,11 +27,16 @@ def __init__(self, field, value, operation): self.value = value self.operation = operation - if self.operation == "in": + if self.operation in ["in", "nin", "inq", "between"]: if not isinstance(self.value, list): raise BadRequestError( - "When using the 'in' operation for a WHERE filter, the values must" - " be in a list format e.g. [1, 2, 3]", + f"When using the {self.operation} operation for a WHERE filter, the" + f" values must be in a list format e.g. [1, 2]", + ) + if self.operation == "between" and len(self.value) != 2: + raise BadRequestError( + "When using the 'between' operation for a WHERE filter, the list" + "must contain two values e.g. [1, 2]", ) diff --git a/datagateway_api/src/datagateway_api/icat/filters.py b/datagateway_api/src/datagateway_api/icat/filters.py index 9877938a..03d79efd 100644 --- a/datagateway_api/src/datagateway_api/icat/filters.py +++ b/datagateway_api/src/datagateway_api/icat/filters.py @@ -49,7 +49,7 @@ def create_filter(self): log.info("Creating condition for ICAT where filter") if self.operation == "eq": where_filter = self.create_condition(self.field, "=", self.value) - elif self.operation == "ne": + elif self.operation in ["ne", "neq"]: where_filter = self.create_condition(self.field, "!=", self.value) elif self.operation == "like": where_filter = self.create_condition(self.field, "like", f"%{self.value}%") @@ -75,7 +75,7 @@ def create_filter(self): where_filter = self.create_condition(self.field, ">", self.value) elif self.operation == "gte": where_filter = self.create_condition(self.field, ">=", self.value) - elif self.operation == "in": + elif self.operation in ["in", "inq"]: # Convert self.value into a string with brackets equivalent to tuple format. # Cannot convert straight to tuple as single element tuples contain a # trailing comma which Python ICAT/JPQL doesn't accept @@ -88,6 +88,25 @@ def create_filter(self): self.value = "(NULL)" where_filter = self.create_condition(self.field, "in", self.value) + elif self.operation == "nin": + # Convert self.value into a string with brackets equivalent to tuple format. + # Cannot convert straight to tuple as single element tuples contain a + # trailing comma which Python ICAT/JPQL doesn't accept + self.value = str(self.value).replace("[", "(").replace("]", ")") + + # DataGateway Search can send requests with blank lists. Adding NULL to the + # filter prevents the API from returning a 500. An empty list will be + # returned instead, equivalent to the DB backend + if self.value == "()": + self.value = "(NULL)" + + where_filter = self.create_condition(self.field, "not in", self.value) + elif self.operation == "between": + where_filter = self.create_condition( + self.field, "between", f"'{self.value[0]}' and '{self.value[1]}'", + ) + elif self.operation == "regexp": + where_filter = self.create_condition(self.field, "regexp", self.value) else: raise FilterError(f"Bad operation given to where filter: {self.operation}") @@ -116,8 +135,7 @@ def create_condition(attribute_name, operator, value): # distinct filter is used in a request jpql_value = ( f"{value}" - if operator == "in" - or operator == "!=" + if operator in ("in", "not in", "!=", "between") or str(value).startswith("UPPER") or "o." in str(value) else f"'{value}'" diff --git a/datagateway_api/src/search_api/filters.py b/datagateway_api/src/search_api/filters.py index 1ebbf579..cd840204 100644 --- a/datagateway_api/src/search_api/filters.py +++ b/datagateway_api/src/search_api/filters.py @@ -3,7 +3,6 @@ import logging from datagateway_api.src.common.date_handler import DateHandler -from datagateway_api.src.common.exceptions import FilterError from datagateway_api.src.datagateway_api.icat.filters import ( PythonICATIncludeFilter, PythonICATLimitFilter, diff --git a/datagateway_api/src/search_api/models.py b/datagateway_api/src/search_api/models.py index 4c3fdc56..f9c8ae9a 100644 --- a/datagateway_api/src/search_api/models.py +++ b/datagateway_api/src/search_api/models.py @@ -1,40 +1,168 @@ -from abc import ABC, abstractclassmethod -from datetime import datetime +from abc import ABC, abstractmethod +from datetime import datetime, timezone +import sys from typing import ClassVar, List, Optional, Union -from pydantic import ( - BaseModel, - Field, - root_validator, - StrictBool, - StrictFloat, - StrictInt, - StrictStr, -) +from dateutil.relativedelta import relativedelta +from pydantic import BaseModel, Field, ValidationError, validator +from pydantic.error_wrappers import ErrorWrapper + +from datagateway_api.src.common.date_handler import DateHandler +from datagateway_api.src.search_api.panosc_mappings import mappings + + +def _is_panosc_entity_field_of_type_list(entity_field): + entity_field_outer_type = entity_field.outer_type_ + if ( + hasattr(entity_field_outer_type, "_name") + and entity_field_outer_type._name == "List" + ): + is_list = True + # The `_name` `outer_type_` attribute was introduced in Python 3.7 so to check + # whether the field is of type list in Python 3.6, we are checking the type of its + # default value. We must ensure that any new list fields that get added in future + # are assigned a list by default. + elif isinstance(entity_field.default, list): + is_list = True + else: + is_list = False + + return is_list + + +def _get_icat_field_value(icat_field_name, icat_data): + icat_field_name = icat_field_name.split(".") + for field_name in icat_field_name: + if isinstance(icat_data, list): + values = [] + for data in icat_data: + value = _get_icat_field_value(field_name, data) + value = [value] if not isinstance(value, list) else value + values.extend(value) + icat_data = values + elif isinstance(icat_data, dict): + icat_data = icat_data[field_name] + + return icat_data class PaNOSCAttribute(ABC, BaseModel): - @abstractclassmethod - def from_icat(self): - pass + @classmethod + @abstractmethod + def from_icat(cls, icat_data, required_related_fields): # noqa: B902, N805 + entity_fields = cls.__fields__ + + entity_data = {} + for entity_field in entity_fields: + # Some fields have aliases so we must use them when creating a model + # instance. If a field does not have an alias then the `alias` property + # holds the name of the field + entity_field_alias = cls.__fields__[entity_field].alias + + entity_name, icat_field_name = mappings.get_icat_mapping( + cls.__name__, entity_field_alias, + ) + + if not isinstance(icat_field_name, list): + icat_field_name = [icat_field_name] + + field_value = None + for field_name in icat_field_name: + try: + field_value = _get_icat_field_value(field_name, icat_data) + if field_value: + break + except KeyError: + # If an icat value cannot be found for the ICAT field name in the + # provided ICAT data then ignore the error. The field name could + # simply be a mapping of an optional PaNOSC entity field so ICAT + # may not return data for it which is fine. It could also be a list + # of mappings which is the case with the `value` field of the + # PaNOSC entity. When this is the case, ICAT only returns data for + # one of the mappings from the list so we can ignore the error. + # This also ignores errors for mandatory fields but this is not a + # problem because pydantic is responsible for validating whether + # data for mandatory fields is missing. + continue + + if not field_value: + continue + + if entity_name != cls.__name__: + # If we are here, it means that the field references another model so + # we have to get hold of its class definition and call its `from_icat` + # method to create an instance of itself with the ICAT data provided. + # Doing this allows for recursion. + data = ( + [field_value] if not isinstance(field_value, list) else field_value + ) + + required_related_fields_for_next_entity = [] + for required_related_field in required_related_fields: + required_related_field = required_related_field.split(".") + if ( + len(required_related_field) > 1 + and entity_field_alias in required_related_field + ): + required_related_fields_for_next_entity.extend( + required_related_field[1:], + ) + + # Get the class of the referenced entity + entity_attr = getattr(sys.modules[__name__], entity_name) + field_value = [ + entity_attr.from_icat(d, required_related_fields_for_next_entity) + for d in data + ] + + if not _is_panosc_entity_field_of_type_list( + cls.__fields__[entity_field], + ) and isinstance(field_value, list): + # If the field does not hold list of values but `field_value` + # is a list, then just get its first element + field_value = field_value[0] + + entity_data[entity_field_alias] = field_value + + for required_related_field in required_related_fields: + required_related_field = required_related_field.split(".")[0] + + if ( + required_related_field in entity_fields + and required_related_field + in cls._related_fields_with_min_cardinality_one + and required_related_field not in entity_data + ): + # If we are here, it means that a related entity, which has a minimum + # cardinality of one, has been specified to be included as part of the + # entity but the relevant ICAT data needed for its creation cannot be + # found in the provided ICAT response. Because of this, a + # `ValidationError` is raised. + error_wrapper = ErrorWrapper( + TypeError("field required"), loc=required_related_field, + ) + raise ValidationError(errors=[error_wrapper], model=cls) + + return cls(**entity_data) class Affiliation(PaNOSCAttribute): """Information about which facility a member is located at""" + _related_fields_with_min_cardinality_one: ClassVar[List[str]] = ["members"] _text_operator_fields: ClassVar[List[str]] = [] - name: Optional[StrictStr] - id_: Optional[StrictStr] = Field(alias="id") - address: Optional[StrictStr] - city: Optional[StrictStr] - country: Optional[StrictStr] + name: Optional[str] = None + id_: Optional[str] = Field(None, alias="id") + address: Optional[str] = None + city: Optional[str] = None + country: Optional[str] = None - members: Optional[List["Member"]] + members: Optional[List["Member"]] = [] @classmethod - def from_icat(cls): - pass + def from_icat(cls, icat_data, required_related_fields): + return super(Affiliation, cls).from_icat(icat_data, required_related_fields) class Dataset(PaNOSCAttribute): @@ -43,24 +171,38 @@ class Dataset(PaNOSCAttribute): and Technique """ + _related_fields_with_min_cardinality_one: ClassVar[List[str]] = [ + "documents", + "techniques", + ] _text_operator_fields: ClassVar[List[str]] = ["title"] - pid: StrictStr - title: StrictStr - is_public: StrictBool = Field(alias="isPublic") + pid: str + title: str + is_public: bool = Field(alias="isPublic") creation_date: datetime = Field(alias="creationDate") - size: Optional[StrictInt] + size: Optional[int] = None + + documents: List["Document"] = [] + techniques: List["Technique"] = [] + instrument: Optional["Instrument"] = None + files: Optional[List["File"]] = [] + parameters: Optional[List["Parameter"]] = [] + samples: Optional[List["Sample"]] = [] + + @validator("is_public", pre=True, always=True) + def set_is_public(cls, value): # noqa: B902, N805 + if not value: + return value - documents: List["Document"] - techniques: List["Technique"] - instrument: Optional["Instrument"] - files: Optional[List["File"]] - parameters: Optional[List["Parameter"]] - samples: Optional[List["Sample"]] + creation_date = DateHandler.str_to_datetime_object(value) + current_datetime = datetime.now(timezone.utc) + three_years_ago = current_datetime - relativedelta(years=3) + return creation_date < three_years_ago @classmethod - def from_icat(cls): - pass + def from_icat(cls, icat_data, required_related_fields): + return super(Dataset, cls).from_icat(icat_data, required_related_fields) class Document(PaNOSCAttribute): @@ -68,77 +210,91 @@ class Document(PaNOSCAttribute): Proposal which includes the dataset or published paper which references the dataset """ + _related_fields_with_min_cardinality_one: ClassVar[List[str]] = ["datasets"] _text_operator_fields: ClassVar[List[str]] = ["title", "summary"] - pid: StrictStr - is_public: StrictBool = Field(alias="isPublic") - type_: StrictStr = Field(alias="type") - title: StrictStr - summary: Optional[StrictStr] - doi: Optional[StrictStr] - start_date: Optional[datetime] = Field(alias="startDate") - end_date: Optional[datetime] = Field(alias="endDate") - release_date: Optional[datetime] = Field(alias="releaseDate") - license_: Optional[StrictStr] = Field(alias="license") - keywords: Optional[List[StrictStr]] - - datasets: List[Dataset] - members: Optional[List["Member"]] - parameters: Optional[List["Parameter"]] + pid: str + is_public: bool = Field(alias="isPublic") + type_: str = Field(alias="type") + title: str + summary: Optional[str] = None + doi: Optional[str] = None + start_date: Optional[datetime] = Field(None, alias="startDate") + end_date: Optional[datetime] = Field(None, alias="endDate") + release_date: Optional[datetime] = Field(None, alias="releaseDate") + license_: Optional[str] = Field(None, alias="license") + keywords: Optional[List[str]] = [] + + datasets: List[Dataset] = [] + members: Optional[List["Member"]] = [] + parameters: Optional[List["Parameter"]] = [] + + @validator("is_public", pre=True, always=True) + def set_is_public(cls, value): # noqa: B902, N805 + if not value: + return value + + creation_date = DateHandler.str_to_datetime_object(value) + current_datetime = datetime.now(timezone.utc) + three_years_ago = current_datetime - relativedelta(years=3) + return creation_date < three_years_ago @classmethod - def from_icat(cls): - pass + def from_icat(cls, icat_data, required_related_fields): + return super(Document, cls).from_icat(icat_data, required_related_fields) class File(PaNOSCAttribute): """Name of file and optionally location""" + _related_fields_with_min_cardinality_one: ClassVar[List[str]] = ["dataset"] _text_operator_fields: ClassVar[List[str]] = ["name"] - id_: StrictStr = Field(alias="id") - name: StrictStr - path: Optional[StrictStr] - size: Optional[StrictInt] + id_: str = Field(alias="id") + name: str + path: Optional[str] = None + size: Optional[int] = None - dataset: Dataset + dataset: Dataset = None @classmethod - def from_icat(cls): - pass + def from_icat(cls, icat_data, required_related_fields): + return super(File, cls).from_icat(icat_data, required_related_fields) class Instrument(PaNOSCAttribute): """Beam line where experiment took place""" + _related_fields_with_min_cardinality_one: ClassVar[List[str]] = [] _text_operator_fields: ClassVar[List[str]] = ["name", "facility"] - pid: StrictStr - name: StrictStr - facility: StrictStr + pid: str + name: str + facility: str - datasets: Optional[List[Dataset]] + datasets: Optional[List[Dataset]] = [] @classmethod - def from_icat(cls): - pass + def from_icat(cls, icat_data, required_related_fields): + return super(Instrument, cls).from_icat(icat_data, required_related_fields) class Member(PaNOSCAttribute): """Proposal team member or paper co-author""" + _related_fields_with_min_cardinality_one: ClassVar[List[str]] = ["document"] _text_operator_fields: ClassVar[List[str]] = [] - id_: StrictStr = Field(alias="id") - role: Optional[StrictStr] = Field(alias="role") + id_: str = Field(alias="id") + role: Optional[str] = Field(None, alias="role") - document: Document - person: Optional["Person"] - affiliation: Optional[Affiliation] + document: Document = None + person: Optional["Person"] = None + affiliation: Optional[Affiliation] = None @classmethod - def from_icat(cls): - pass + def from_icat(cls, icat_data, required_related_fields): + return super(Member, cls).from_icat(icat_data, required_related_fields) class Parameter(PaNOSCAttribute): @@ -147,15 +303,26 @@ class Parameter(PaNOSCAttribute): Note: a parameter is either related to a dataset or a document, but not both. """ + _related_fields_with_min_cardinality_one: ClassVar[List[str]] = [] _text_operator_fields: ClassVar[List[str]] = [] - id_: StrictStr = Field(alias="id") - name: StrictStr - value: Union[StrictFloat, StrictInt, StrictStr] - unit: Optional[StrictStr] + id_: str = Field(alias="id") + name: str + value: Union[float, int, str] + unit: Optional[str] = None - dataset: Optional[Dataset] - document: Optional[Document] + dataset: Optional[Dataset] = None + document: Optional[Document] = None + + """ + Validator commented as it was decided to be disabled for the time being. The Data + Model states that a Parameter must be related to a Dataset or Document, however + considering that there is not a Parameter endpoint, it means that a Parameter can + only be included via Dataset or Document. It's unclear why anyone would query for + a Dataset or Document that includes Parameters which in turn includes a Dataset or + Document that are the same as the top level ones. To avoid errors being raised + as a result of Parameters not containing ICAT data for a Dataset or Document, the + validator has been disabled. @root_validator(skip_on_failure=True) def validate_dataset_and_document(cls, values): # noqa: B902, N805 @@ -167,60 +334,64 @@ def validate_dataset_and_document(cls, values): # noqa: B902, N805 values["Document"] = None return values + """ @classmethod - def from_icat(cls): - pass + def from_icat(cls, icat_data, required_related_fields): + return super(Parameter, cls).from_icat(icat_data, required_related_fields) class Person(PaNOSCAttribute): """Human who carried out experiment""" + _related_fields_with_min_cardinality_one: ClassVar[List[str]] = [] _text_operator_fields: ClassVar[List[str]] = [] - id_: StrictStr = Field(alias="id") - full_name: StrictStr = Field(alias="fullName") - orcid: Optional[StrictStr] - researcher_id: Optional[StrictStr] = Field(alias="researcherId") - first_name: Optional[StrictStr] = Field(alias="firstName") - last_name: Optional[StrictStr] = Field(alias="lastName") + id_: str = Field(alias="id") + full_name: str = Field(alias="fullName") + orcid: Optional[str] = None + researcher_id: Optional[str] = Field(None, alias="researcherId") + first_name: Optional[str] = Field(None, alias="firstName") + last_name: Optional[str] = Field(None, alias="lastName") - members: Optional[List[Member]] + members: Optional[List[Member]] = [] @classmethod - def from_icat(cls): - pass + def from_icat(cls, icat_data, required_related_fields): + return super(Person, cls).from_icat(icat_data, required_related_fields) class Sample(PaNOSCAttribute): """Extract of material used in the experiment""" + _related_fields_with_min_cardinality_one: ClassVar[List[str]] = [] _text_operator_fields: ClassVar[List[str]] = ["name", "description"] - name: StrictStr - pid: StrictStr - description: Optional[StrictStr] + name: str + pid: str + description: Optional[str] = None - datasets: Optional[List[Dataset]] + datasets: Optional[List[Dataset]] = [] @classmethod - def from_icat(cls): - pass + def from_icat(cls, icat_data, required_related_fields): + return super(Sample, cls).from_icat(icat_data, required_related_fields) class Technique(PaNOSCAttribute): """Common name of scientific method used""" + _related_fields_with_min_cardinality_one: ClassVar[List[str]] = [] _text_operator_fields: ClassVar[List[str]] = ["name"] - pid: StrictStr - name: StrictStr + pid: str + name: str - datasets: Optional[List[Dataset]] + datasets: Optional[List[Dataset]] = [] @classmethod - def from_icat(cls): - pass + def from_icat(cls, icat_data, required_related_fields): + return super(Technique, cls).from_icat(icat_data, required_related_fields) # The below models reference other models that may not be defined during their diff --git a/datagateway_api/src/search_api/panosc_mappings.py b/datagateway_api/src/search_api/panosc_mappings.py index c5f1ffe2..695c0245 100644 --- a/datagateway_api/src/search_api/panosc_mappings.py +++ b/datagateway_api/src/search_api/panosc_mappings.py @@ -49,7 +49,7 @@ def get_icat_mapping(self, panosc_entity_name, field_name): ) try: - icat_mapping = mappings.mappings[panosc_entity_name][field_name] + icat_mapping = self.mappings[panosc_entity_name][field_name] log.debug("ICAT mapping/translation found: %s", icat_mapping) except KeyError as e: raise FilterError(f"Bad PaNOSC to ICAT mapping: {e.args}") diff --git a/datagateway_api/src/search_api/query_filter_factory.py b/datagateway_api/src/search_api/query_filter_factory.py index eecf3f7b..79a558ad 100644 --- a/datagateway_api/src/search_api/query_filter_factory.py +++ b/datagateway_api/src/search_api/query_filter_factory.py @@ -245,15 +245,13 @@ def get_include_filter(include_filter_input, entity_name): scope_query_filter, included_entity, ) if isinstance(scope_query_filter, SearchAPIIncludeFilter): - for included_entity_pointer in range( - len(scope_query_filter.included_filters), + for i, included_filter in enumerate( + scope_query_filter.included_filters, ): nested_include = True - included_filter = scope_query_filter.included_filters[ - included_entity_pointer - ] + scope_query_filter.panosc_entity_name = entity_name scope_query_filter.included_filters[ - included_entity_pointer + i ] = f"{included_entity}.{included_filter}" query_filters.extend(scope_query_filters) diff --git a/noxfile.py b/noxfile.py index 4a97e124..60f2ddd6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -85,5 +85,16 @@ def safety(session): @nox.session(python=["3.6", "3.7", "3.8", "3.9"], reuse_venv=True) def tests(session): args = session.posargs + # Installing setuptools that will work with `2to3` which is used when building + # `python-icat` < 1.0. 58.0.0 removes support of this tool during builds: + # https://setuptools.pypa.io/en/latest/history.html#v58-0-0 + # Ideally this would be done within `pyproject.toml` but specifying `setuptools` as + # a dependency requires Poetry 1.2: + # https://github.com/python-poetry/poetry/issues/4511#issuecomment-922420457 + # Currently, only a pre-release exists for Poetry 1.2. Testing on the pre-release + # version didn't fix the `2to3` issue when building Python ICAT, perhaps because + # Python ICAT isn't built on the downgraded version for some reason? + session.run("poetry", "run", "pip", "uninstall", "-y", "setuptools") + session.run("poetry", "run", "pip", "install", "setuptools<58.0.0") session.run("poetry", "install", external=True) session.run("pytest", *args) diff --git a/poetry.lock b/poetry.lock index 12bedca2..a7f7eafc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -18,12 +18,12 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["PyYAML (>=3.10)", "prance[osv] (>=0.11)", "marshmallow (>=2.19.2)", "pytest", "mock", "flake8 (==3.7.9)", "flake8-bugbear (==20.1.4)", "pre-commit (>=1.20,<3.0)", "tox"] +tests = ["PyYAML (>=3.10)", "prance[osv] (>=0.11)", "marshmallow (>=2.19.2)", "pytest", "mock"] docs = ["marshmallow (>=2.19.2)", "pyyaml (==5.3)", "sphinx (==2.4.1)", "sphinx-issues (==1.2.0)", "sphinx-rtd-theme (==0.4.3)"] lint = ["flake8 (==3.7.9)", "flake8-bugbear (==20.1.4)", "pre-commit (>=1.20,<3.0)"] -tests = ["PyYAML (>=3.10)", "prance[osv] (>=0.11)", "marshmallow (>=2.19.2)", "pytest", "mock"] -validation = ["prance[osv] (>=0.11)"] +dev = ["PyYAML (>=3.10)", "prance[osv] (>=0.11)", "marshmallow (>=2.19.2)", "pytest", "mock", "flake8 (==3.7.9)", "flake8-bugbear (==20.1.4)", "pre-commit (>=1.20,<3.0)", "tox"] yaml = ["PyYAML (>=3.10)"] +validation = ["prance[osv] (>=0.11)"] [[package]] name = "appdirs" @@ -50,10 +50,10 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] [[package]] name = "bandit" @@ -198,11 +198,11 @@ python-versions = ">=3.6" cffi = ">=1.12" [package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] -docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] sdist = ["setuptools_rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] [[package]] @@ -409,8 +409,8 @@ Jinja2 = ">=3.0" Werkzeug = ">=2.0" [package.extras] -async = ["asgiref (>=3.2)"] dotenv = ["python-dotenv"] +async = ["asgiref (>=3.2)"] [[package]] name = "flask-cors" @@ -528,8 +528,8 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -perf = ["ipython"] testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +perf = ["ipython"] [[package]] name = "iniconfig" @@ -666,8 +666,8 @@ click = ">=7" six = "*" [package.extras] -coverage = ["pytest-cov"] testing = ["mock", "pytest", "pytest-rerunfailures"] +coverage = ["pytest-cov"] [[package]] name = "pkginfo" @@ -692,8 +692,8 @@ python-versions = ">=3.6" importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] -dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +dev = ["pre-commit", "tox"] [[package]] name = "pprintpp" @@ -862,12 +862,12 @@ requests = ">=2.25.0" requests-toolbelt = ">=0.9.1" [package.extras] -autocompletion = ["argcomplete (>=1.10.0,<2)"] yaml = ["PyYaml (>=5.2)"] +autocompletion = ["argcomplete (>=1.10.0,<2)"] [[package]] name = "python-icat" -version = "0.20.0" +version = "0.21.0" description = "Python interface to ICAT and IDS" category = "main" optional = false @@ -894,10 +894,10 @@ tomlkit = "0.7.0" twine = ">=3,<4" [package.extras] -dev = ["tox", "isort", "black"] +test = ["coverage (>=5,<6)", "pytest (>=5,<6)", "pytest-xdist (>=1,<2)", "pytest-mock (>=2,<3)", "responses (==0.13.3)", "mock (==1.3.0)"] docs = ["Sphinx (==1.3.6)"] mypy = ["mypy", "types-requests"] -test = ["coverage (>=5,<6)", "pytest (>=5,<6)", "pytest-xdist (>=1,<2)", "pytest-mock (>=2,<3)", "responses (==0.13.3)", "mock (==1.3.0)"] +dev = ["tox", "isort", "black"] [[package]] name = "pytz" @@ -962,8 +962,8 @@ idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} urllib3 = ">=1.21.1,<1.27" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] [[package]] name = "requests-toolbelt" @@ -1065,25 +1065,25 @@ greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platfo importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [package.extras] -aiomysql = ["greenlet (!=0.4.17)", "aiomysql"] -aiosqlite = ["typing_extensions (!=3.10.0.1)", "greenlet (!=0.4.17)", "aiosqlite"] -asyncio = ["greenlet (!=0.4.17)"] -asyncmy = ["greenlet (!=0.4.17)", "asyncmy (>=0.2.3)"] -mariadb_connector = ["mariadb (>=1.0.1)"] mssql = ["pyodbc"] -mssql_pymssql = ["pymssql"] -mssql_pyodbc = ["pyodbc"] mypy = ["sqlalchemy2-stubs", "mypy (>=0.910)"] -mysql = ["mysqlclient (>=1.4.0,<2)", "mysqlclient (>=1.4.0)"] -mysql_connector = ["mysql-connector-python"] -oracle = ["cx_oracle (>=7,<8)", "cx_oracle (>=7)"] postgresql = ["psycopg2 (>=2.7)"] postgresql_asyncpg = ["greenlet (!=0.4.17)", "asyncpg"] -postgresql_pg8000 = ["pg8000 (>=1.16.6)"] -postgresql_psycopg2binary = ["psycopg2-binary"] postgresql_psycopg2cffi = ["psycopg2cffi"] +mysql_connector = ["mysql-connector-python"] pymysql = ["pymysql (<1)", "pymysql"] +mssql_pymssql = ["pymssql"] +oracle = ["cx_oracle (>=7,<8)", "cx_oracle (>=7)"] +postgresql_pg8000 = ["pg8000 (>=1.16.6)"] +aiosqlite = ["typing_extensions (!=3.10.0.1)", "greenlet (!=0.4.17)", "aiosqlite"] +asyncmy = ["greenlet (!=0.4.17)", "asyncmy (>=0.2.3)"] +postgresql_psycopg2binary = ["psycopg2-binary"] +mysql = ["mysqlclient (>=1.4.0,<2)", "mysqlclient (>=1.4.0)"] sqlcipher = ["sqlcipher3-binary"] +aiomysql = ["greenlet (!=0.4.17)", "aiomysql"] +mariadb_connector = ["mariadb (>=1.0.1)"] +mssql_pyodbc = ["pyodbc"] +asyncio = ["greenlet (!=0.4.17)"] [[package]] name = "stevedore" @@ -1149,9 +1149,9 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] -notebook = ["ipywidgets (>=6)"] telegram = ["requests"] +notebook = ["ipywidgets (>=6)"] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] [[package]] name = "twine" @@ -1238,7 +1238,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<4.0" -content-hash = "821a222ad69359ec266eff8d281ba0b3fe78c5249ff0d703f9359afb4c89efba" +content-hash = "489cc7708ea5d15d5c47bc7f4c21298761b4250ea93abdfcc7f1b7e85c7785d3" [metadata.files] aniso8601 = [ @@ -1525,6 +1525,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97e5306482182170ade15c4b0d8386ded995a07d7cc2ca8f27958d34d6736497"}, {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6a36bb9474218c7a5b27ae476035497a6990e21d04c279884eb10d9b290f1b1"}, {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abb7a75ed8b968f3061327c433a0fbd17b729947b400747c334a9c29a9af6c58"}, + {file = "greenlet-1.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b336501a05e13b616ef81ce329c0e09ac5ed8c732d9ba7e3e983fcc1a9e86965"}, {file = "greenlet-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:14d4f3cd4e8b524ae9b8aa567858beed70c392fdec26dbdb0a8a418392e71708"}, {file = "greenlet-1.1.2-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:17ff94e7a83aa8671a25bf5b59326ec26da379ace2ebc4411d690d80a7fbcf23"}, {file = "greenlet-1.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9f3cba480d3deb69f6ee2c1825060177a22c7826431458c697df88e6aeb3caee"}, @@ -1537,6 +1538,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9d29ca8a77117315101425ec7ec2a47a22ccf59f5593378fc4077ac5b754fce"}, {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21915eb821a6b3d9d8eefdaf57d6c345b970ad722f856cd71739493ce003ad08"}, {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eff9d20417ff9dcb0d25e2defc2574d10b491bf2e693b4e491914738b7908168"}, + {file = "greenlet-1.1.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b8c008de9d0daba7b6666aa5bbfdc23dcd78cafc33997c9b7741ff6353bafb7f"}, {file = "greenlet-1.1.2-cp36-cp36m-win32.whl", hash = "sha256:32ca72bbc673adbcfecb935bb3fb1b74e663d10a4b241aaa2f5a75fe1d1f90aa"}, {file = "greenlet-1.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f0214eb2a23b85528310dad848ad2ac58e735612929c8072f6093f3585fd342d"}, {file = "greenlet-1.1.2-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:b92e29e58bef6d9cfd340c72b04d74c4b4e9f70c9fa7c78b674d1fec18896dc4"}, @@ -1545,6 +1547,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e12bdc622676ce47ae9abbf455c189e442afdde8818d9da983085df6312e7a1"}, {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c790abda465726cfb8bb08bd4ca9a5d0a7bd77c7ac1ca1b839ad823b948ea28"}, {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f276df9830dba7a333544bd41070e8175762a7ac20350786b322b714b0e654f5"}, + {file = "greenlet-1.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c5d5b35f789a030ebb95bff352f1d27a93d81069f2adb3182d99882e095cefe"}, {file = "greenlet-1.1.2-cp37-cp37m-win32.whl", hash = "sha256:64e6175c2e53195278d7388c454e0b30997573f3f4bd63697f88d855f7a6a1fc"}, {file = "greenlet-1.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b11548073a2213d950c3f671aa88e6f83cda6e2fb97a8b6317b1b5b33d850e06"}, {file = "greenlet-1.1.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:9633b3034d3d901f0a46b7939f8c4d64427dfba6bbc5a36b1a67364cf148a1b0"}, @@ -1553,6 +1556,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e859fcb4cbe93504ea18008d1df98dee4f7766db66c435e4882ab35cf70cac43"}, {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00e44c8afdbe5467e4f7b5851be223be68adb4272f44696ee71fe46b7036a711"}, {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec8c433b3ab0419100bd45b47c9c8551248a5aee30ca5e9d399a0b57ac04651b"}, + {file = "greenlet-1.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2bde6792f313f4e918caabc46532aa64aa27a0db05d75b20edfc5c6f46479de2"}, {file = "greenlet-1.1.2-cp38-cp38-win32.whl", hash = "sha256:288c6a76705dc54fba69fbcb59904ae4ad768b4c768839b8ca5fdadec6dd8cfd"}, {file = "greenlet-1.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:8d2f1fb53a421b410751887eb4ff21386d119ef9cde3797bf5e7ed49fb51a3b3"}, {file = "greenlet-1.1.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:166eac03e48784a6a6e0e5f041cfebb1ab400b394db188c48b3a84737f505b67"}, @@ -1561,6 +1565,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1692f7d6bc45e3200844be0dba153612103db241691088626a33ff1f24a0d88"}, {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7227b47e73dedaa513cdebb98469705ef0d66eb5a1250144468e9c3097d6b59b"}, {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ff61ff178250f9bb3cd89752df0f1dd0e27316a8bd1465351652b1b4a4cdfd3"}, + {file = "greenlet-1.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0051c6f1f27cb756ffc0ffbac7d2cd48cb0362ac1736871399a739b2885134d3"}, {file = "greenlet-1.1.2-cp39-cp39-win32.whl", hash = "sha256:f70a9e237bb792c7cc7e44c531fd48f5897961701cdaa06cf22fc14965c496cf"}, {file = "greenlet-1.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:013d61294b6cd8fe3242932c1c5e36e5d1db2c8afb58606c5a67efce62c1f5fd"}, {file = "greenlet-1.1.2.tar.gz", hash = "sha256:e30f5ea4ae2346e62cedde8794a56858a67b878dd79f7df76a0767e356b1744a"}, @@ -1784,7 +1789,7 @@ python-gitlab = [ {file = "python_gitlab-2.10.1-py3-none-any.whl", hash = "sha256:581a219759515513ea9399e936ed7137437cfb681f52d2641626685c492c999d"}, ] python-icat = [ - {file = "python-icat-0.20.0.tar.gz", hash = "sha256:d09e85f0269ac5f8f6a43f2365eb269b532e611a5eaa3ab23ad7f742ee2bfa25"}, + {file = "python-icat-0.21.0.tar.gz", hash = "sha256:fd4d1515b8075677ee3672274a55d6ca287ce1cfa4e17b6b25371904764be999"}, ] python-semantic-release = [ {file = "python-semantic-release-7.19.2.tar.gz", hash = "sha256:8ca0e5f72d31e7b0603b95caad6fb2d5315483ac1fadd86648771966d9ec6f2c"}, diff --git a/pyproject.toml b/pyproject.toml index 62c96806..4ae98f75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datagateway-api" -version = "3.1.1" +version = "3.5.1" description = "ICAT API to interface with the DataGateway" license = "Apache-2.0" readme = "README.md" @@ -26,7 +26,7 @@ Flask-Cors = "3.0.9" apispec = "3.3.0" flask-swagger-ui = "3.25.0" PyYAML = "5.4" -python-icat = "0.20.0" +python-icat = "0.21.0" suds-community = "^0.8.4" py-object-pool = "^1.1" cachetools = "^4.2.1" diff --git a/test/datagateway_api/icat/filters/test_where_filter.py b/test/datagateway_api/icat/filters/test_where_filter.py index 71a0cca9..07eabeb9 100644 --- a/test/datagateway_api/icat/filters/test_where_filter.py +++ b/test/datagateway_api/icat/filters/test_where_filter.py @@ -10,7 +10,8 @@ class TestICATWhereFilter: "operation, value, expected_condition_value", [ pytest.param("eq", 5, ["%s = '5'"], id="equal"), - pytest.param("ne", 5, ["%s != 5"], id="not equal"), + pytest.param("ne", 5, ["%s != 5"], id="not equal (ne)"), + pytest.param("neq", 5, ["%s != 5"], id="not equal (neq)"), pytest.param("like", 5, ["%s like '%%5%%'"], id="like"), pytest.param("ilike", 5, ["UPPER(%s) like UPPER('%%5%%')"], id="ilike"), pytest.param("nlike", 5, ["%s not like '%%5%%'"], id="not like"), @@ -21,8 +22,20 @@ class TestICATWhereFilter: pytest.param("lte", 5, ["%s <= '5'"], id="less than or equal"), pytest.param("gt", 5, ["%s > '5'"], id="greater than"), pytest.param("gte", 5, ["%s >= '5'"], id="greater than or equal"), - pytest.param("in", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list"), - pytest.param("in", [], ["%s in (NULL)"], id="empty list"), + pytest.param( + "in", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list (in)", + ), + pytest.param("in", [], ["%s in (NULL)"], id="in empty list (in)"), + pytest.param( + "inq", [1, 2, 3, 4], ["%s in (1, 2, 3, 4)"], id="in a list (inq)", + ), + pytest.param("inq", [], ["%s in (NULL)"], id="in empty list (inq)"), + pytest.param( + "nin", [1, 2, 3, 4], ["%s not in (1, 2, 3, 4)"], id="not in a list", + ), + pytest.param("nin", [], ["%s not in (NULL)"], id="not in empty list"), + pytest.param("between", [1, 2], ["%s between '1' and '2'"], id="between"), + pytest.param("regexp", "^Test", ["%s regexp '^Test'"], id="regexp"), ], ) def test_valid_operations( @@ -33,9 +46,25 @@ def test_valid_operations( assert icat_query.conditions == {"id": expected_condition_value} - def test_invalid_in_operation(self, icat_query): + @pytest.mark.parametrize( + "operation, value", + [ + pytest.param("in", "1, 2, 3, 4, 5", id="in a list (in)"), + pytest.param("inq", "1, 2, 3, 4, 5", id="in a list (inq)"), + pytest.param("nin", "1, 2, 3, 4, 5", id="nin"), + pytest.param("between", "1, 2, 3, 4, 5", id="between - string value"), + pytest.param("between", [], id="between - empty list"), + pytest.param( + "between", [1], id="between - list with less than two elements", + ), + pytest.param( + "between", [1, 2, 3], id="between - list with more than two elements", + ), + ], + ) + def test_invalid_operations_raise_bad_request_error(self, operation, value): with pytest.raises(BadRequestError): - PythonICATWhereFilter("id", "1, 2, 3, 4, 5", "in") + PythonICATWhereFilter("id", value, operation) def test_invalid_operation(self, icat_query): test_filter = PythonICATWhereFilter("id", 10, "non") diff --git a/test/search_api/filters/test_search_api_include_filter.py b/test/search_api/filters/test_search_api_include_filter.py index cf207484..e0ba2e69 100644 --- a/test/search_api/filters/test_search_api_include_filter.py +++ b/test/search_api/filters/test_search_api_include_filter.py @@ -53,16 +53,6 @@ def test_valid_apply_filter_single(self): " iu.user", id="Document.members.person", ), - # TODO - Below test fails because dataset/investigation parameters issue on - # mapping - pytest.param( - SearchAPIIncludeFilter(["dataset.parameters.document"], "File"), - "File", - "SELECT o FROM Datafile o INCLUDE o.dataset AS d, d.parameters AS p," - " p.investigation", - id="File.dataset.parameters.document", - marks=pytest.mark.skip, - ), pytest.param( SearchAPIIncludeFilter(["dataset.samples.datasets"], "File"), "File", diff --git a/test/search_api/test_models.py b/test/search_api/test_models.py new file mode 100644 index 00000000..df6aa84b --- /dev/null +++ b/test/search_api/test_models.py @@ -0,0 +1,725 @@ +from datetime import datetime, timezone + +from pydantic import ValidationError +import pytest + +from datagateway_api.src.common.date_handler import DateHandler +import datagateway_api.src.search_api.models as models + + +AFFILIATION_ICAT_DATA = { + "id": 1, + "name": "Test name", + "fullReference": "Test fullReference", +} + +DATASET_ICAT_DATA = { + "endDate": "2000-12-31 00:00:00+00:00", + "complete": True, + "name": "Test name", + "id": 1, + "modTime": "2000-12-31 00:00:00+00:00", + "location": "Test location", + "modId": "Test modId", + "description": "Test description", + "createId": "Test createId", + "createTime": "2000-12-31 00:00:00+00:00", + "doi": "Test doi", + "startDate": "2000-12-31 00:00:00+00:00", +} + +DATASET_PARAMETER_ICAT_DATA = { + "error": 1.0, + "stringValue": "Test stringValue", + "id": 1, + "numericValue": None, + "modTime": "2000-12-31 00:00:00+00:00", + "rangeTop": 1.0, + "modId": "Test modId", + "createId": "Test createId", + "createTime": "2000-12-31 00:00:00+00:00", + "rangeBottom": 1.0, + "dateTimeValue": None, +} + +DATAFILE_ICAT_DATA = { + "name": "Test name", + "id": 1, + "datafileModTime": "2000-12-31 00:00:00+00:00", + "modTime": "2000-12-31 00:00:00+00:00", + "fileSize": 1234, + "location": "Test location", + "modId": "Test modId", + "description": "Test description", + "createId": "Test createId", + "createTime": "2000-12-31 00:00:00+00:00", + "doi": "Test doi", + "datafileCreateTime": "2000-12-31 00:00:00+00:00", + "checksum": "Test checksum", +} + +FACILITY_ICAT_DATA = { + "daysUntilRelease": 1, + "name": "Test name", + "id": 1, + "modTime": "2000-12-31 00:00:00+00:00", + "url": None, + "fullName": None, + "modId": "Test modId", + "description": "Test description", + "createId": "Test createId", + "createTime": "2000-12-31 00:00:00+00:00", +} + +INSTRUMENT_ICAT_DATA = { + "type": "Test type", + "pid": None, + "name": "Test name", + "id": 1, + "modTime": "2000-12-31 00:00:00+00:00", + "url": "Test url", + "fullName": "Test fullName", + "modId": "Test modId", + "description": "Test description", + "createId": "Test createId", + "createTime": "2000-12-31 00:00:00+00:00", +} + +INVESTIGATION_ICAT_DATA = { + "endDate": "2000-12-31 00:00:00+00:00", + "name": "Test name", + "releaseDate": str(datetime.now(timezone.utc)), + "id": 1, + "modTime": "2000-12-31 00:00:00+00:00", + "modId": "Test modId", + "createId": "Test createId", + "summary": "Test summary", + "visitId": "Test visitId", + "createTime": "2000-12-31 00:00:00+00:00", + "doi": "Test doi", + "startDate": "2000-12-31 00:00:00+00:00", + "title": "Test title", +} + +INVESTIGATION_PARAMETER_ICAT_DATA = DATASET_PARAMETER_ICAT_DATA.copy() +INVESTIGATION_PARAMETER_ICAT_DATA["stringValue"] = None +INVESTIGATION_PARAMETER_ICAT_DATA["dateTimeValue"] = "2000-12-31 00:00:00+00:00" + +INVESTIGATION_TYPE_ICAT_DATA = { + "modId": "Test modId", + "description": "Test description", + "createId": "Test createId", + "name": "Test name", + "createTime": "2000-12-31 00:00:00+00:00", + "id": 1, + "modTime": "2000-12-31 00:00:00+00:00", +} + +INVESTIGATION_USER_ICAT_DATA = { + "id": 1, + "modId": "Test modId", + "createId": "Test createId", + "createTime": "2000-12-31 00:00:00+00:00", + "role": "Test role", + "modTime": "2000-12-31 00:00:00+00:00", +} + +KEYWORD_ICAT_DATA = { + "modId": "Test modId", + "createId": "Test createId", + "name": "Test name", + "createTime": "2000-12-31 00:00:00+00:00", + "id": 1, + "modTime": "2000-12-31 00:00:00+00:00", +} + +PARAMETER_TYPE_ICAT_DATA = { + "pid": None, + "verified": True, + "unitsFullName": "Test unitsFullName", + "enforced": False, + "maximumNumericValue": 1.0, + "name": "Test name", + "modId": "Test modId", + "minimumNumericValue": 1.0, + "createId": "Test createId", + "applicableToDataCollection": False, + "applicableToInvestigation": False, + "applicableToDatafile": True, + "units": "Test units", + "id": 1, + "modTime": "2000-12-31 00:00:00+00:00", + "applicableToDataset": True, + "description": "Test description", + "valueType": "Test valueType", + "createTime": "2000-12-31 00:00:00+00:00", + "applicableToSample": False, +} + +SAMPLE_ICAT_DATA = { + "pid": "None", + "modId": "Test modId", + "createId": "Test createId", + "name": "Test name", + "createTime": "2000-12-31 00:00:00+00:00", + "id": 1, + "modTime": "2000-12-31 00:00:00+00:00", +} + +TECHNIQUE_ICAT_DATA = { + "name": "Test name", + "pid": "Test pid", + "description": "Test description", +} + +USER_ICAT_DATA = { + "email": "Test email", + "name": "Test name", + "id": 1, + "modTime": "2000-12-31 00:00:00+00:00", + "familyName": None, + "modId": "Test modId", + "fullName": "Test fullName", + "orcidId": "Test orcidId", + "createId": "Test createId", + "createTime": "2000-12-31 00:00:00+00:00", + "affiliation": None, + "givenName": None, +} + + +AFFILIATION_PANOSC_DATA = { + "name": AFFILIATION_ICAT_DATA["name"], + "id": str(AFFILIATION_ICAT_DATA["id"]), + "address": AFFILIATION_ICAT_DATA["fullReference"], + "city": None, + "country": None, + "members": [], +} + +DATASET_PANOSC_DATA = { + "pid": DATASET_ICAT_DATA["doi"], + "title": DATASET_ICAT_DATA["name"], + "creationDate": DateHandler.str_to_datetime_object( + DATASET_ICAT_DATA["createTime"], + ), + "isPublic": True, + "size": None, + "documents": [], + "techniques": [], + "instrument": None, + "files": [], + "parameters": [], + "samples": [], +} + +DOCUMENT_PANOSC_DATA = { + "pid": INVESTIGATION_ICAT_DATA["doi"], + "isPublic": False, + "type": INVESTIGATION_TYPE_ICAT_DATA["name"], + "title": INVESTIGATION_ICAT_DATA["name"], + "summary": INVESTIGATION_ICAT_DATA["summary"], + "doi": INVESTIGATION_ICAT_DATA["doi"], + "startDate": DateHandler.str_to_datetime_object( + INVESTIGATION_ICAT_DATA["startDate"], + ), + "endDate": DateHandler.str_to_datetime_object(INVESTIGATION_ICAT_DATA["endDate"]), + "releaseDate": DateHandler.str_to_datetime_object( + INVESTIGATION_ICAT_DATA["releaseDate"], + ), + "license": None, + "keywords": [KEYWORD_ICAT_DATA["name"]], + "datasets": [], + "members": [], + "parameters": [], +} + +FILE_PANOSC_DATA = { + "id": str(DATAFILE_ICAT_DATA["id"]), + "name": DATAFILE_ICAT_DATA["name"], + "path": DATAFILE_ICAT_DATA["location"], + "size": DATAFILE_ICAT_DATA["fileSize"], + "dataset": None, +} + +INSTRUMENT_PANOSC_DATA = { + "pid": str(INSTRUMENT_ICAT_DATA["id"]), + "name": INSTRUMENT_ICAT_DATA["name"], + "facility": FACILITY_ICAT_DATA["name"], + "datasets": [], +} + +MEMBER_PANOSC_DATA = { + "id": str(INVESTIGATION_USER_ICAT_DATA["id"]), + "role": INVESTIGATION_USER_ICAT_DATA["role"], + "document": None, + "person": None, + "affiliation": None, +} + +PARAMETER_PANOSC_DATA = { + "id": str(INVESTIGATION_PARAMETER_ICAT_DATA["id"]), + "name": PARAMETER_TYPE_ICAT_DATA["name"], + "value": INVESTIGATION_PARAMETER_ICAT_DATA["dateTimeValue"], + "unit": PARAMETER_TYPE_ICAT_DATA["units"], + "dataset": None, + "document": None, +} + +PERSON_PANOSC_DATA = { + "id": str(USER_ICAT_DATA["id"]), + "fullName": USER_ICAT_DATA["fullName"], + "orcid": USER_ICAT_DATA["orcidId"], + "researcherId": None, + "firstName": USER_ICAT_DATA["givenName"], + "lastName": USER_ICAT_DATA["familyName"], + "members": [], +} + +SAMPLE_PANOSC_DATA = { + "name": SAMPLE_ICAT_DATA["name"], + "pid": SAMPLE_ICAT_DATA["pid"], + "description": PARAMETER_TYPE_ICAT_DATA["description"], + "datasets": [], +} + +TECHNIQUE_PANOSC_DATA = { + "pid": TECHNIQUE_ICAT_DATA["pid"], + "name": TECHNIQUE_ICAT_DATA["name"], + "datasets": [], +} + + +class TestModels: + def test_from_icat_affiliation_entity_without_data_for_related_entities(self): + affiliation_entity = models.Affiliation.from_icat(AFFILIATION_ICAT_DATA, []) + + assert affiliation_entity.dict(by_alias=True) == AFFILIATION_PANOSC_DATA + + def test_from_icat_affiliation_entity_with_data_for_all_related_entities(self): + expected_entity_data = AFFILIATION_PANOSC_DATA.copy() + expected_entity_data["members"] = [MEMBER_PANOSC_DATA] + + icat_data = AFFILIATION_ICAT_DATA.copy() + icat_data["user"] = { + "user": {"investigationUsers": [INVESTIGATION_USER_ICAT_DATA]}, + } + + affiliation_entity = models.Affiliation.from_icat(icat_data, ["members"]) + + assert affiliation_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_dataset_entity_without_data_for_related_entities(self): + dataset_entity = models.Dataset.from_icat(DATASET_ICAT_DATA, []) + + assert dataset_entity.dict(by_alias=True) == DATASET_PANOSC_DATA + + def test_from_icat_dataset_entity_with_data_for_mandatory_related_entities(self): + expected_entity_data = DATASET_PANOSC_DATA.copy() + expected_entity_data["documents"] = [DOCUMENT_PANOSC_DATA] + expected_entity_data["techniques"] = [ + TECHNIQUE_PANOSC_DATA, + TECHNIQUE_PANOSC_DATA, + ] + + icat_data = DATASET_ICAT_DATA.copy() + icat_data["investigation"] = INVESTIGATION_ICAT_DATA.copy() + icat_data["investigation"]["type"] = INVESTIGATION_TYPE_ICAT_DATA + icat_data["investigation"]["keywords"] = [KEYWORD_ICAT_DATA] + icat_data["datasetTechniques"] = [ + {"technique": TECHNIQUE_ICAT_DATA}, + {"technique": TECHNIQUE_ICAT_DATA}, + ] + + dataset_entity = models.Dataset.from_icat( + icat_data, ["documents", "techniques"], + ) + + assert dataset_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_dataset_entity_with_data_for_all_related_entities(self): + expected_entity_data = DATASET_PANOSC_DATA.copy() + expected_entity_data["documents"] = [DOCUMENT_PANOSC_DATA] + expected_entity_data["techniques"] = [TECHNIQUE_PANOSC_DATA] + expected_entity_data["instrument"] = INSTRUMENT_PANOSC_DATA + expected_entity_data["files"] = [FILE_PANOSC_DATA, FILE_PANOSC_DATA] + expected_entity_data["parameters"] = [PARAMETER_PANOSC_DATA.copy()] + expected_entity_data["parameters"][0]["value"] = DATASET_PARAMETER_ICAT_DATA[ + "stringValue" + ] + expected_entity_data["samples"] = [SAMPLE_PANOSC_DATA] + + icat_data = DATASET_ICAT_DATA.copy() + icat_data["investigation"] = INVESTIGATION_ICAT_DATA.copy() + icat_data["investigation"]["type"] = INVESTIGATION_TYPE_ICAT_DATA + icat_data["investigation"]["keywords"] = [KEYWORD_ICAT_DATA] + icat_data["datasetTechniques"] = [{"technique": TECHNIQUE_ICAT_DATA}] + icat_data["datasetInstruments"] = [ + {"instrument": INSTRUMENT_ICAT_DATA.copy()}, + {"instrument": INSTRUMENT_ICAT_DATA.copy()}, + ] + icat_data["datasetInstruments"][0]["instrument"][ + "facility" + ] = FACILITY_ICAT_DATA + icat_data["datasetInstruments"][1]["instrument"][ + "facility" + ] = FACILITY_ICAT_DATA + icat_data["datafiles"] = [DATAFILE_ICAT_DATA, DATAFILE_ICAT_DATA] + icat_data["parameters"] = [DATASET_PARAMETER_ICAT_DATA.copy()] + icat_data["parameters"][0]["type"] = PARAMETER_TYPE_ICAT_DATA + icat_data["sample"] = SAMPLE_ICAT_DATA.copy() + icat_data["sample"]["parameters"] = [ + {"type": PARAMETER_TYPE_ICAT_DATA}, + {"type": PARAMETER_TYPE_ICAT_DATA}, + ] + + dataset_entity = models.Dataset.from_icat( + icat_data, + ["documents", "techniques", "instrument", "files", "parameters", "samples"], + ) + + assert dataset_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_document_entity_without_data_for_related_entities(self): + icat_data = INVESTIGATION_ICAT_DATA.copy() + icat_data["type"] = INVESTIGATION_TYPE_ICAT_DATA + icat_data["keywords"] = [KEYWORD_ICAT_DATA] + + document_entity = models.Document.from_icat(icat_data, []) + + assert document_entity.dict(by_alias=True) == DOCUMENT_PANOSC_DATA + + def test_from_icat_document_entity_with_data_for_mandatory_related_entities(self): + expected_entity_data = DOCUMENT_PANOSC_DATA.copy() + expected_entity_data["datasets"] = [DATASET_PANOSC_DATA, DATASET_PANOSC_DATA] + + icat_data = INVESTIGATION_ICAT_DATA.copy() + icat_data["type"] = INVESTIGATION_TYPE_ICAT_DATA + icat_data["keywords"] = [KEYWORD_ICAT_DATA] + icat_data["datasets"] = [DATASET_ICAT_DATA, DATASET_ICAT_DATA] + + document_entity = models.Document.from_icat(icat_data, ["datasets"]) + + assert document_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_document_entity_with_data_for_all_related_entities(self): + expected_entity_data = DOCUMENT_PANOSC_DATA.copy() + expected_entity_data["datasets"] = [DATASET_PANOSC_DATA, DATASET_PANOSC_DATA] + expected_entity_data["members"] = [MEMBER_PANOSC_DATA] + expected_entity_data["parameters"] = [PARAMETER_PANOSC_DATA] + + icat_data = INVESTIGATION_ICAT_DATA.copy() + icat_data["type"] = INVESTIGATION_TYPE_ICAT_DATA + icat_data["keywords"] = [KEYWORD_ICAT_DATA] + icat_data["datasets"] = [DATASET_ICAT_DATA, DATASET_ICAT_DATA] + icat_data["investigationUsers"] = [INVESTIGATION_USER_ICAT_DATA] + icat_data["parameters"] = [INVESTIGATION_PARAMETER_ICAT_DATA.copy()] + icat_data["parameters"][0]["type"] = PARAMETER_TYPE_ICAT_DATA + + document_entity = models.Document.from_icat(icat_data, ["datasets"]) + + assert document_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_file_entity_without_data_for_related_entities(self): + file_entity = models.File.from_icat(DATAFILE_ICAT_DATA, []) + + assert file_entity.dict(by_alias=True) == FILE_PANOSC_DATA + + def test_from_icat_file_entity_with_data_for_all_related_entities(self): + expected_entity_data = FILE_PANOSC_DATA.copy() + expected_entity_data["dataset"] = DATASET_PANOSC_DATA + + icat_data = DATAFILE_ICAT_DATA.copy() + icat_data["dataset"] = DATASET_ICAT_DATA + + file_entity = models.File.from_icat(icat_data, []) + + assert file_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_instrument_entity_without_data_for_related_entities(self): + icat_data = INSTRUMENT_ICAT_DATA.copy() + icat_data["facility"] = FACILITY_ICAT_DATA + + instrument_entity = models.Instrument.from_icat(icat_data, []) + + assert instrument_entity.dict(by_alias=True) == INSTRUMENT_PANOSC_DATA + + def test_from_icat_instrument_entity_with_data_for_all_related_entities(self): + expected_entity_data = INSTRUMENT_PANOSC_DATA.copy() + expected_entity_data["datasets"] = [DATASET_PANOSC_DATA] + + icat_data = INSTRUMENT_ICAT_DATA.copy() + icat_data["facility"] = FACILITY_ICAT_DATA + icat_data["datasetInstruments"] = [{"dataset": DATASET_ICAT_DATA.copy()}] + + instrument_entity = models.Instrument.from_icat(icat_data, ["datasets"]) + + assert instrument_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_member_entity_without_data_for_related_entities(self): + member_entity = models.Member.from_icat(INVESTIGATION_USER_ICAT_DATA, []) + + assert member_entity.dict(by_alias=True) == MEMBER_PANOSC_DATA + + def test_from_icat_member_entity_with_data_for_mandatory_related_entities(self): + expected_entity_data = MEMBER_PANOSC_DATA.copy() + expected_entity_data["document"] = DOCUMENT_PANOSC_DATA + + icat_data = INVESTIGATION_USER_ICAT_DATA.copy() + icat_data["investigation"] = INVESTIGATION_ICAT_DATA.copy() + icat_data["investigation"]["type"] = INVESTIGATION_TYPE_ICAT_DATA + icat_data["investigation"]["keywords"] = [KEYWORD_ICAT_DATA] + + member_entity = models.Member.from_icat(icat_data, ["document"]) + + assert member_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_member_entity_with_data_for_all_related_entities(self): + expected_entity_data = MEMBER_PANOSC_DATA.copy() + expected_entity_data["document"] = DOCUMENT_PANOSC_DATA + expected_entity_data["person"] = PERSON_PANOSC_DATA + expected_entity_data["affiliation"] = AFFILIATION_PANOSC_DATA + + icat_data = INVESTIGATION_USER_ICAT_DATA.copy() + icat_data["investigation"] = INVESTIGATION_ICAT_DATA.copy() + icat_data["investigation"]["type"] = INVESTIGATION_TYPE_ICAT_DATA + icat_data["investigation"]["keywords"] = [KEYWORD_ICAT_DATA] + icat_data["user"] = USER_ICAT_DATA.copy() + icat_data["user"]["dataPublicationUsers"] = [ + {"affiliations": [AFFILIATION_ICAT_DATA]}, + ] + + member_entity = models.Member.from_icat( + icat_data, ["document", "person", "affiliation"], + ) + + assert member_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_parameter_entity_without_data_for_related_entities(self): + icat_data = INVESTIGATION_PARAMETER_ICAT_DATA.copy() + icat_data["type"] = PARAMETER_TYPE_ICAT_DATA + + parameter_entity = models.Parameter.from_icat(icat_data, []) + + assert parameter_entity.dict(by_alias=True) == PARAMETER_PANOSC_DATA + + def test_from_icat_parameter_entity_with_investigation_parameter_data(self): + expected_entity_data = PARAMETER_PANOSC_DATA.copy() + expected_entity_data["document"] = DOCUMENT_PANOSC_DATA + + icat_data = INVESTIGATION_PARAMETER_ICAT_DATA.copy() + icat_data["type"] = PARAMETER_TYPE_ICAT_DATA + icat_data["investigation"] = INVESTIGATION_ICAT_DATA.copy() + icat_data["investigation"]["type"] = INVESTIGATION_TYPE_ICAT_DATA + icat_data["investigation"]["keywords"] = [KEYWORD_ICAT_DATA] + + parameter_entity = models.Parameter.from_icat(icat_data, ["document"]) + + assert parameter_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_parameter_entity_with_dataset_parameter_data(self): + expected_entity_data = PARAMETER_PANOSC_DATA.copy() + expected_entity_data["value"] = DATASET_PARAMETER_ICAT_DATA["stringValue"] + expected_entity_data["dataset"] = DATASET_PANOSC_DATA + + icat_data = DATASET_PARAMETER_ICAT_DATA.copy() + icat_data["dataset"] = DATASET_ICAT_DATA + icat_data["type"] = PARAMETER_TYPE_ICAT_DATA + + parameter_entity = models.Parameter.from_icat(icat_data, ["dataset"]) + + assert parameter_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_person_entity_without_data_for_related_entities(self): + person_entity = models.Person.from_icat(USER_ICAT_DATA, []) + + assert person_entity.dict(by_alias=True) == PERSON_PANOSC_DATA + + def test_from_icat_person_entity_with_data_for_all_related_entities(self): + expected_entity_data = PERSON_PANOSC_DATA.copy() + expected_entity_data["members"] = [MEMBER_PANOSC_DATA, MEMBER_PANOSC_DATA] + + icat_data = USER_ICAT_DATA.copy() + icat_data["investigationUsers"] = [ + INVESTIGATION_USER_ICAT_DATA, + INVESTIGATION_USER_ICAT_DATA, + ] + + person_entity = models.Person.from_icat(icat_data, ["members"]) + + assert person_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_sample_entity_without_data_for_related_entities(self): + icat_data = SAMPLE_ICAT_DATA.copy() + icat_data["parameters"] = [ + {"type": PARAMETER_TYPE_ICAT_DATA}, + ] + sample_entity = models.Sample.from_icat(icat_data, []) + + assert sample_entity.dict(by_alias=True) == SAMPLE_PANOSC_DATA + + def test_from_icat_sample_entity_with_data_for_all_related_entities(self): + expected_entity_data = SAMPLE_PANOSC_DATA.copy() + expected_entity_data["datasets"] = [DATASET_PANOSC_DATA, DATASET_PANOSC_DATA] + + icat_data = SAMPLE_ICAT_DATA.copy() + icat_data["parameters"] = [ + {"type": PARAMETER_TYPE_ICAT_DATA}, + ] + icat_data["datasets"] = [DATASET_ICAT_DATA, DATASET_ICAT_DATA] + + sample_entity = models.Sample.from_icat(icat_data, ["datasets"]) + + assert sample_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_technique_entity_without_data_for_related_entities(self): + technique_entity = models.Technique.from_icat(TECHNIQUE_ICAT_DATA, []) + + assert technique_entity.dict(by_alias=True) == TECHNIQUE_PANOSC_DATA + + def test_from_icat_technique_entity_with_data_for_all_related_entities(self): + expected_entity_data = TECHNIQUE_PANOSC_DATA.copy() + expected_entity_data["datasets"] = [DATASET_PANOSC_DATA] + + icat_data = TECHNIQUE_ICAT_DATA.copy() + icat_data["datasetTechniques"] = [{"dataset": DATASET_ICAT_DATA}] + + technique_entity = models.Technique.from_icat(icat_data, ["datasets"]) + + assert technique_entity.dict(by_alias=True) == expected_entity_data + + def test_from_icat_multiple_and_nested_relations(self): + expected_entity_data = DOCUMENT_PANOSC_DATA.copy() + expected_entity_data["keywords"] = [] + expected_entity_data["datasets"] = [DATASET_PANOSC_DATA.copy()] + expected_entity_data["datasets"][0]["instrument"] = INSTRUMENT_PANOSC_DATA + expected_entity_data["datasets"][0]["files"] = [ + FILE_PANOSC_DATA, + FILE_PANOSC_DATA, + ] + expected_entity_data["datasets"][0]["parameters"] = [ + PARAMETER_PANOSC_DATA.copy(), + ] + expected_entity_data["datasets"][0]["parameters"][0][ + "value" + ] = DATASET_PARAMETER_ICAT_DATA["stringValue"] + expected_entity_data["datasets"][0]["samples"] = [SAMPLE_PANOSC_DATA.copy()] + expected_entity_data["datasets"][0]["samples"][0]["description"] = None + expected_entity_data["members"] = [MEMBER_PANOSC_DATA.copy()] + expected_entity_data["members"][0]["affiliation"] = AFFILIATION_PANOSC_DATA + expected_entity_data["members"][0]["person"] = PERSON_PANOSC_DATA + expected_entity_data["parameters"] = [PARAMETER_PANOSC_DATA.copy()] + expected_entity_data["parameters"][0]["document"] = DOCUMENT_PANOSC_DATA + + icat_data = INVESTIGATION_ICAT_DATA.copy() + icat_data["type"] = INVESTIGATION_TYPE_ICAT_DATA + icat_data["datasets"] = [DATASET_ICAT_DATA.copy()] + icat_data["datasets"][0]["datasetInstruments"] = [ + {"instrument": INSTRUMENT_ICAT_DATA.copy()}, + ] + icat_data["datasets"][0]["datasetInstruments"][0]["instrument"][ + "facility" + ] = FACILITY_ICAT_DATA + icat_data["datasets"][0]["datafiles"] = [DATAFILE_ICAT_DATA, DATAFILE_ICAT_DATA] + icat_data["datasets"][0]["parameters"] = [DATASET_PARAMETER_ICAT_DATA.copy()] + icat_data["datasets"][0]["parameters"][0]["type"] = PARAMETER_TYPE_ICAT_DATA + icat_data["datasets"][0]["sample"] = SAMPLE_ICAT_DATA + icat_data["investigationUsers"] = [INVESTIGATION_USER_ICAT_DATA.copy()] + icat_data["investigationUsers"][0]["user"] = USER_ICAT_DATA.copy() + icat_data["investigationUsers"][0]["user"]["dataPublicationUsers"] = [ + {"affiliations": [AFFILIATION_ICAT_DATA]}, + ] + icat_data["parameters"] = [INVESTIGATION_PARAMETER_ICAT_DATA.copy()] + icat_data["parameters"][0]["type"] = PARAMETER_TYPE_ICAT_DATA + icat_data["parameters"][0]["investigation"] = INVESTIGATION_ICAT_DATA.copy() + icat_data["parameters"][0]["investigation"][ + "type" + ] = INVESTIGATION_TYPE_ICAT_DATA + icat_data["parameters"][0]["investigation"]["keywords"] = [KEYWORD_ICAT_DATA] + + relations = [ + "datasets.instrument", + "datasets.files", + "datasets.parameters", + "datasets.samples", + "members.affiliation", + "members.person", + "parameters.document", + ] + + document_entity = models.Document.from_icat(icat_data, relations) + + assert document_entity.dict(by_alias=True) == expected_entity_data + + @pytest.mark.parametrize( + "panosc_entity_name, icat_data, required_related_fields", + [ + pytest.param( + "Affiliation", + AFFILIATION_ICAT_DATA, + ["members"], + id="Affiliation - without data for mandatory related entity", + ), + pytest.param( + "Dataset", {}, [], id="Dataset - without data for mandatory fields", + ), + pytest.param( + "Dataset", + DATASET_ICAT_DATA, + ["documents"], + id="Dataset - without data for mandatory related entity", + ), + pytest.param( + "Document", {}, [], id="Document - without data for mandatory fields", + ), + pytest.param( + "Document", + INVESTIGATION_ICAT_DATA, + ["dataset"], + id="Document - without data for mandatory related entity", + ), + pytest.param( + "File", {}, [], id="File - without data for mandatory fields", + ), + pytest.param( + "File", + DATAFILE_ICAT_DATA, + ["dataset"], + id="File - without data for mandatory related entity", + ), + pytest.param( + "Instrument", + {}, + [], + id="Instrument - without data for mandatory fields", + ), + pytest.param( + "Member", {}, [], id="Member - without data for mandatory fields", + ), + pytest.param( + "Member", + INVESTIGATION_USER_ICAT_DATA, + ["document"], + id="Member - without data for mandatory related entity", + ), + pytest.param( + "Parameter", {}, [], id="Parameter - without data for mandatory fields", + ), + pytest.param( + "Person", {}, [], id="Person - without data for mandatory fields", + ), + pytest.param( + "Sample", {}, [], id="Sample - without data for mandatory fields", + ), + pytest.param( + "Technique", {}, [], id="Technique - without data for mandatory fields", + ), + ], + ) + def test_from_icat_raises_validation_error( + self, panosc_entity_name, icat_data, required_related_fields, + ): + with pytest.raises(ValidationError): + getattr(models, panosc_entity_name).from_icat( + icat_data, required_related_fields, + ) diff --git a/test/search_api/test_search_api_query_filter_factory.py b/test/search_api/test_search_api_query_filter_factory.py index 1df109da..25a4b9e2 100644 --- a/test/search_api/test_search_api_query_filter_factory.py +++ b/test/search_api/test_search_api_query_filter_factory.py @@ -142,10 +142,10 @@ def test_valid_where_filter_text_operator( id="Multiple conditions (three), property values with no operator", ), pytest.param( - {"filter": {"where": {"and": [{"value": {"lt": 50}}]}}}, - "Parameter", + {"filter": {"where": {"and": [{"size": {"lt": 50}}]}}}, + "File", [], - [SearchAPIWhereFilter("value", 50, "lt")], + [SearchAPIWhereFilter("size", 50, "lt")], "and", id="Single condition, property value with operator", ), @@ -154,15 +154,15 @@ def test_valid_where_filter_text_operator( "filter": { "where": { "and": [ - {"name": {"like": "Test name"}}, - {"value": {"gte": 275}}, + {"role": {"like": "Test role"}}, + {"size": {"gte": 275}}, ], }, }, }, - "Parameter", - [SearchAPIWhereFilter("name", "Test name", "like")], - [SearchAPIWhereFilter("value", 275, "gte")], + "Member", + [SearchAPIWhereFilter("role", "Test role", "like")], + [SearchAPIWhereFilter("size", 275, "gte")], "and", id="Multiple conditions (two), property values with operator", ), @@ -172,18 +172,18 @@ def test_valid_where_filter_text_operator( "where": { "and": [ {"name": {"like": "Test name"}}, - {"value": {"gte": 275}}, - {"unit": {"nlike": "Test unit"}}, + {"size": {"gte": 275}}, + {"path": {"nlike": "Test path"}}, ], }, }, }, - "Parameter", + "File", [ SearchAPIWhereFilter("name", "Test name", "like"), - SearchAPIWhereFilter("value", 275, "gte"), + SearchAPIWhereFilter("size", 275, "gte"), ], - [SearchAPIWhereFilter("unit", "Test unit", "nlike")], + [SearchAPIWhereFilter("path", "Test path", "nlike")], "and", id="Multiple conditions (three), property values with operator", ), @@ -371,10 +371,10 @@ def test_valid_where_filter_with_and_boolean_operator( id="Multiple conditions (three), property values with no operator", ), pytest.param( - {"filter": {"where": {"or": [{"value": {"lt": 50}}]}}}, - "Parameter", + {"filter": {"where": {"or": [{"size": {"lt": 50}}]}}}, + "File", [], - [SearchAPIWhereFilter("value", 50, "lt")], + [SearchAPIWhereFilter("size", 50, "lt")], "or", id="Single condition, property value with operator", ), @@ -384,14 +384,14 @@ def test_valid_where_filter_with_and_boolean_operator( "where": { "or": [ {"name": {"like": "Test name"}}, - {"value": {"gte": 275}}, + {"size": {"gte": 275}}, ], }, }, }, - "Parameter", + "File", [SearchAPIWhereFilter("name", "Test name", "like")], - [SearchAPIWhereFilter("value", 275, "gte")], + [SearchAPIWhereFilter("size", 275, "gte")], "or", id="Multiple conditions (two), property values with operator", ), @@ -401,18 +401,18 @@ def test_valid_where_filter_with_and_boolean_operator( "where": { "or": [ {"name": {"like": "Test name"}}, - {"value": {"gte": 275}}, - {"unit": {"nlike": "Test unit"}}, + {"size": {"gte": 275}}, + {"path": {"nlike": "Test path"}}, ], }, }, }, - "Parameter", + "File", [ SearchAPIWhereFilter("name", "Test name", "like"), - SearchAPIWhereFilter("value", 275, "gte"), + SearchAPIWhereFilter("size", 275, "gte"), ], - [SearchAPIWhereFilter("unit", "Test unit", "nlike")], + [SearchAPIWhereFilter("path", "Test path", "nlike")], "or", id="Multiple conditions (three), property values with operator", ), @@ -1714,6 +1714,7 @@ def test_valid_include_filter_with_where_filter_in_scope( assert len(filters) == expected_length for test_filter in filters: if isinstance(test_filter, SearchAPIIncludeFilter): + assert test_filter.panosc_entity_name == test_entity_name for expected_include in expected_included_entities: assert test_filter.included_filters == expected_include expected_included_entities.remove(expected_include)