From 9b4c9f855d3bc9dc69ebf094f1e40c5ae7e23227 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt <897972+cutoffthetop@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:24:18 +0200 Subject: [PATCH] feature/mx-1670 backend api connector (#263) # PR Context - as a prep for MX-1382, MX-1669, MX-1406, MX-1416 and MX-1581 # Added - added `BackendApiConnector` methods to cover all current (and near future) endpoints: `fetch_extracted_items`, `fetch_merged_items`, `get_merged_item`, `preview_merged_item` and `get_rule_set` - complete the list of exported names in `models` and `types` modules # Deprecated - deprecated `BackendApiConnector.post_models` in favor of `post_extracted_items` # Fixed - added the `rki/mex` user-agent to all requests of the HTTPConnector --- CHANGELOG.md | 9 ++ mex/common/backend_api/connector.py | 203 ++++++++++++++++++++++-- mex/common/backend_api/models.py | 48 +++++- mex/common/connector/http.py | 11 +- mex/common/models/__init__.py | 39 ++++- mex/common/sinks/backend_api.py | 4 +- mex/common/types/__init__.py | 4 + tests/backend_api/conftest.py | 18 +++ tests/backend_api/test_connector.py | 229 ++++++++++++++++++++++++---- tests/conftest.py | 50 +++++- tests/connector/test_http.py | 24 ++- tests/sinks/test_backend_api.py | 14 +- 12 files changed, 588 insertions(+), 65 deletions(-) create mode 100644 tests/backend_api/conftest.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cde8c4d3..efe63886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,16 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- added `BackendApiConnector` methods to cover all current (and near future) endpoints: + `fetch_extracted_items`, `fetch_merged_items`, `get_merged_item`, + `preview_merged_item` and `get_rule_set` +- complete the list of exported names in `models` and `types` modules + ### Changes ### Deprecated +- deprecated `BackendApiConnector.post_models` in favor of `post_extracted_items` + ### Removed - containerize section from release pipeline ### Fixed +- added the `rki/mex` user-agent to all requests of the HTTPConnector + ### Security ## [0.35.0] - 2024-08-20 diff --git a/mex/common/backend_api/connector.py b/mex/common/backend_api/connector.py index 0b68331b..4db67cfe 100644 --- a/mex/common/backend_api/connector.py +++ b/mex/common/backend_api/connector.py @@ -1,8 +1,23 @@ +from typing import cast from urllib.parse import urljoin -from mex.common.backend_api.models import BulkInsertResponse +from requests.exceptions import HTTPError + +from mex.common.backend_api.models import ( + ExtractedItemsRequest, + ExtractedItemsResponse, + IdentifiersResponse, + MergedItemsResponse, + MergedModelTypeAdapter, + RuleSetResponseTypeAdapter, +) from mex.common.connector import HTTPConnector -from mex.common.models import AnyExtractedModel +from mex.common.models import ( + AnyExtractedModel, + AnyMergedModel, + AnyRuleSetRequest, + AnyRuleSetResponse, +) from mex.common.settings import BaseSettings from mex.common.types import AnyExtractedIdentifier @@ -27,21 +42,191 @@ def _set_url(self) -> None: self.url = urljoin(str(settings.backend_api_url), self.API_VERSION) def post_models( - self, models: list[AnyExtractedModel] + self, + extracted_items: list[AnyExtractedModel], ) -> list[AnyExtractedIdentifier]: - """Post models to Backend API in a bulk insertion request. + """Post extracted models to the backend in bulk. Args: - models: Extracted or merged models to post + extracted_items: Extracted models to post Raises: - HTTPError: If insert was not accepted, crashes or times out + HTTPError: If post was not accepted, crashes or times out Returns: Identifiers of posted extracted models """ + # XXX deprecated method, please use `post_extracted_models` instead + return cast( + list[AnyExtractedIdentifier], + self.post_extracted_items(extracted_items).identifiers, + ) + + def post_extracted_items( + self, + extracted_items: list[AnyExtractedModel], + ) -> IdentifiersResponse: + """Post extracted items to the backend in bulk. + + Args: + extracted_items: Extracted items to post + + Raises: + HTTPError: If post was not accepted, crashes or times out + + Returns: + Response model from the endpoint + """ + response = self.request( + method="POST", + endpoint="ingest", + payload=ExtractedItemsRequest(items=extracted_items), + ) + return IdentifiersResponse.model_validate(response) + + def fetch_extracted_items( + self, + query_string: str | None, + stable_target_id: str | None, + entity_type: list[str] | None, + skip: int, + limit: int, + ) -> ExtractedItemsResponse: + """Fetch extracted items that match the given set of filters. + + Args: + query_string: Full-text search query + stable_target_id: The item's stableTargetId + entity_type: The item's entityType + skip: How many items to skip for pagination + limit: How many items to return in one page + + Raises: + HTTPError: If search was not accepted, crashes or times out + + Returns: + One page of extracted items and the total count that was matched + """ + response = self.request( + method="GET", + endpoint="extracted-item", + params={ + "q": query_string, + "stableTargetId": stable_target_id, + "entityType": entity_type, + "skip": str(skip), + "limit": str(limit), + }, + ) + return ExtractedItemsResponse.model_validate(response) + + def fetch_merged_items( + self, + query_string: str | None, + entity_type: list[str] | None, + skip: int, + limit: int, + ) -> MergedItemsResponse: + """Fetch merged items that match the given set of filters. + + Args: + query_string: Full-text search query + entity_type: The item's entityType + skip: How many items to skip for pagination + limit: How many items to return in one page + + Raises: + HTTPError: If search was not accepted, crashes or times out + + Returns: + One page of merged items and the total count that was matched + """ + # XXX this endpoint will only return faux merged items for now (MX-1382) + response = self.request( + method="GET", + endpoint="merged-item", + params={ + "q": query_string, + "entityType": entity_type, + "skip": str(skip), + "limit": str(limit), + }, + ) + return MergedItemsResponse.model_validate(response) + + def get_merged_item( + self, + stable_target_id: str, + ) -> AnyMergedModel: + """Return one merged item for the given `stableTargetId`. + + Args: + stable_target_id: The merged item's identifier + + Raises: + MExError: If no merged item was found + + Returns: + A single merged item + """ + # XXX stop-gap until the backend has a proper get merged item endpoint (MX-1669) + response = self.request( + method="GET", + endpoint="merged-item", + params={ + "stableTargetId": stable_target_id, + "limit": "1", + }, + ) + response_model = MergedItemsResponse.model_validate(response) + try: + return response_model.items[0] + except IndexError: + raise HTTPError("merged item was not found") from None + + def preview_merged_item( + self, + stable_target_id: str, + rule_set: AnyRuleSetRequest, + ) -> AnyMergedModel: + """Return a preview for merging the given rule-set with stored extracted items. + + Args: + stable_target_id: The extracted items' `stableTargetId` + rule_set: A rule-set to use for previewing + + Raises: + HTTPError: If preview produces errors, crashes or times out + + Returns: + A single merged item + """ + # XXX experimental method until the backend has a preview endpoint (MX-1406) + response = self.request( + method="GET", + endpoint=f"preview-item/{stable_target_id}", + payload=rule_set, + ) + return MergedModelTypeAdapter.validate_python(response) + + def get_rule_set( + self, + stable_target_id: str, + ) -> AnyRuleSetResponse: + """Return a triple of rules for the given `stableTargetId`. + + Args: + stable_target_id: The merged item's identifier + + Raises: + HTTPError: If no rule-set was found + + Returns: + A set of three rules + """ + # XXX experimental method until the backend has a rule-set endpoint (MX-1416) response = self.request( - method="POST", endpoint="ingest", payload={"items": models} + method="GET", + endpoint=f"rule-set/{stable_target_id}", ) - insert_response = BulkInsertResponse.model_validate(response) - return insert_response.identifiers + return RuleSetResponseTypeAdapter.validate_python(response) diff --git a/mex/common/backend_api/models.py b/mex/common/backend_api/models.py index adc59785..2a7057a0 100644 --- a/mex/common/backend_api/models.py +++ b/mex/common/backend_api/models.py @@ -1,8 +1,46 @@ -from mex.common.models import BaseModel -from mex.common.types import AnyExtractedIdentifier +from typing import Annotated +from pydantic import Field, TypeAdapter -class BulkInsertResponse(BaseModel): - """Response body for the bulk ingestion endpoint.""" +from mex.common.models import ( + AnyExtractedModel, + AnyMergedModel, + AnyRuleSetResponse, + BaseModel, +) +from mex.common.types import Identifier - identifiers: list[AnyExtractedIdentifier] + +class ExtractedItemsRequest(BaseModel): + """Request model for a list of extracted items.""" + + items: list[AnyExtractedModel] + + +class ExtractedItemsResponse(BaseModel): + """Response model for a list of extracted items including a total count.""" + + items: list[AnyExtractedModel] + total: int + + +class MergedItemsResponse(BaseModel): + """Response model for a list of merged items including a total count.""" + + items: list[AnyMergedModel] + total: int + + +class IdentifiersResponse(BaseModel): + """Response models for a list of identifiers.""" + + identifiers: list[Identifier] + + +MergedModelTypeAdapter: TypeAdapter[AnyMergedModel] = TypeAdapter( + Annotated[AnyMergedModel, Field(discriminator="entityType")] +) +RuleSetResponseTypeAdapter: TypeAdapter[AnyRuleSetResponse] = TypeAdapter( + Annotated[AnyRuleSetResponse, Field(discriminator="entityType")] +) +BulkInsertResponse = IdentifiersResponse # deprecated diff --git a/mex/common/connector/http.py b/mex/common/connector/http.py index 8e037954..e6e1ef12 100644 --- a/mex/common/connector/http.py +++ b/mex/common/connector/http.py @@ -1,5 +1,6 @@ import json from abc import abstractmethod +from collections.abc import Mapping from typing import Any, Literal, cast import backoff @@ -48,7 +49,7 @@ def request( method: Literal["OPTIONS", "POST", "GET", "PUT", "DELETE"], endpoint: str | None = None, payload: Any = None, - params: dict[str, str] | None = None, + params: Mapping[str, list[str] | str | None] | None = None, **kwargs: Any, ) -> dict[str, Any]: """Prepare and send a request with error handling and payload de/serialization. @@ -77,10 +78,10 @@ def request( if not kwargs.get("headers"): kwargs.setdefault("headers", {}) kwargs["headers"].setdefault("Accept", "application/json") + kwargs["headers"].setdefault("User-Agent", "rki/mex") if payload: kwargs["data"] = json.dumps(payload, cls=MExEncoder) - kwargs["headers"].setdefault("User-Agent", "rki/mex") # Send request response = self._send_request(method, url, params, **kwargs) @@ -114,7 +115,11 @@ def request( ) @backoff.on_exception(backoff.fibo, RequestException, max_tries=6) def _send_request( - self, method: str, url: str, params: dict[str, str] | None, **kwargs: Any + self, + method: str, + url: str, + params: Mapping[str, list[str] | str | None] | None, + **kwargs: Any, ) -> Response: """Send the response with advanced retrying rules.""" return self.session.request(method, url, params, **kwargs) diff --git a/mex/common/models/__init__.py b/mex/common/models/__init__.py index a6ff4b90..403029e8 100644 --- a/mex/common/models/__init__.py +++ b/mex/common/models/__init__.py @@ -191,6 +191,10 @@ ) __all__ = ( + "AccessPlatformRuleSetRequest", + "AccessPlatformRuleSetResponse", + "ActivityRuleSetRequest", + "ActivityRuleSetResponse", "ADDITIVE_MODEL_CLASSES_BY_NAME", "ADDITIVE_MODEL_CLASSES", "AdditiveAccessPlatform", @@ -206,11 +210,16 @@ "AdditiveVariable", "AdditiveVariableGroup", "AnyAdditiveModel", + "AnyBaseModel", "AnyExtractedModel", "AnyMergedModel", "AnyPreventiveModel", "AnyRuleModel", + "AnyRuleSetRequest", + "AnyRuleSetResponse", "AnySubtractiveModel", + "BASE_MODEL_CLASSES_BY_NAME", + "BASE_MODEL_CLASSES", "BaseAccessPlatform", "BaseActivity", "BaseContactPoint", @@ -223,6 +232,10 @@ "BaseResource", "BaseVariable", "BaseVariableGroup", + "ContactPointRuleSetRequest", + "ContactPointRuleSetResponse", + "DistributionRuleSetRequest", + "DistributionRuleSetResponse", "EXTRACTED_MODEL_CLASSES_BY_NAME", "EXTRACTED_MODEL_CLASSES", "ExtractedAccessPlatform", @@ -234,10 +247,14 @@ "ExtractedOrganizationalUnit", "ExtractedPerson", "ExtractedPrimarySource", + "ExtractedPrimarySourceIdentifier", "ExtractedResource", "ExtractedVariable", "ExtractedVariableGroup", "FILTER_MODEL_BY_EXTRACTED_CLASS_NAME", + "generate_entity_filter_schema", + "generate_mapping_schema", + "GenericFieldInfo", "MAPPING_MODEL_BY_EXTRACTED_CLASS_NAME", "MERGED_MODEL_CLASSES_BY_NAME", "MERGED_MODEL_CLASSES", @@ -249,14 +266,22 @@ "MergedOrganization", "MergedOrganizationalUnit", "MergedPerson", - "GenericFieldInfo", "MergedPrimarySource", + "MergedPrimarySourceIdentifier", "MergedResource", "MergedVariable", "MergedVariableGroup", "MEX_PRIMARY_SOURCE_IDENTIFIER_IN_PRIMARY_SOURCE", "MEX_PRIMARY_SOURCE_IDENTIFIER", "MEX_PRIMARY_SOURCE_STABLE_TARGET_ID", + "OrganizationalUnitRuleSetRequest", + "OrganizationalUnitRuleSetResponse", + "OrganizationRuleSetRequest", + "OrganizationRuleSetResponse", + "PersonRuleSetRequest", + "PersonRuleSetResponse", + "PREVENTIVE_MODEL_CLASSES_BY_NAME", + "PREVENTIVE_MODEL_CLASSES", "PreventiveAccessPlatform", "PreventiveActivity", "PreventiveContactPoint", @@ -269,8 +294,16 @@ "PreventiveRule", "PreventiveVariable", "PreventiveVariableGroup", + "PrimarySourceRuleSetRequest", + "PrimarySourceRuleSetResponse", + "ResourceRuleSetRequest", + "ResourceRuleSetResponse", "RULE_MODEL_CLASSES_BY_NAME", "RULE_MODEL_CLASSES", + "RULE_SET_REQUEST_CLASSES_BY_NAME", + "RULE_SET_REQUEST_CLASSES", + "RULE_SET_RESPONSE_CLASSES_BY_NAME", + "RULE_SET_RESPONSE_CLASSES", "SUBTRACTIVE_MODEL_CLASSES_BY_NAME", "SUBTRACTIVE_MODEL_CLASSES", "SubtractiveAccessPlatform", @@ -285,6 +318,10 @@ "SubtractiveRule", "SubtractiveVariable", "SubtractiveVariableGroup", + "VariableGroupRuleSetRequest", + "VariableGroupRuleSetResponse", + "VariableRuleSetRequest", + "VariableRuleSetResponse", ) MEX_PRIMARY_SOURCE_IDENTIFIER = ExtractedPrimarySourceIdentifier("00000000000001") diff --git a/mex/common/sinks/backend_api.py b/mex/common/sinks/backend_api.py index 1871c066..69550d6a 100644 --- a/mex/common/sinks/backend_api.py +++ b/mex/common/sinks/backend_api.py @@ -1,4 +1,5 @@ from collections.abc import Generator, Iterable +from typing import cast from mex.common.backend_api.connector import BackendApiConnector from mex.common.logging import watch @@ -23,4 +24,5 @@ def post_to_backend_api( connector = BackendApiConnector.get() for chunk in grouper(chunk_size, models): model_list = [model for model in chunk if model is not None] - yield from connector.post_models(model_list) + response = connector.post_extracted_items(model_list) + yield from cast(list[AnyExtractedIdentifier], response.identifiers) diff --git a/mex/common/types/__init__.py b/mex/common/types/__init__.py index d804b8b7..8df713b9 100644 --- a/mex/common/types/__init__.py +++ b/mex/common/types/__init__.py @@ -78,6 +78,8 @@ "DataProcessingState", "DataType", "Email", + "EXTRACTED_IDENTIFIER_CLASSES_BY_NAME", + "EXTRACTED_IDENTIFIER_CLASSES", "ExtractedAccessPlatformIdentifier", "ExtractedActivityIdentifier", "ExtractedContactPointIdentifier", @@ -98,6 +100,8 @@ "Link", "LinkLanguage", "LiteralStringType", + "MERGED_IDENTIFIER_CLASSES_BY_NAME", + "MERGED_IDENTIFIER_CLASSES", "MergedAccessPlatformIdentifier", "MergedActivityIdentifier", "MergedContactPointIdentifier", diff --git a/tests/backend_api/conftest.py b/tests/backend_api/conftest.py new file mode 100644 index 00000000..6096e441 --- /dev/null +++ b/tests/backend_api/conftest.py @@ -0,0 +1,18 @@ +from unittest.mock import MagicMock, Mock + +import pytest +from pytest import MonkeyPatch + +from mex.common.backend_api.connector import BackendApiConnector + + +@pytest.fixture() +def mocked_backend(monkeypatch: MonkeyPatch) -> MagicMock: + """Return the mocked request dispatch method of backend connector.""" + mocked_send_request = MagicMock( + spec=BackendApiConnector._send_request, + return_value=Mock(json=MagicMock(return_value={})), + name="mocked_backend_send_request", + ) + monkeypatch.setattr(BackendApiConnector, "_send_request", mocked_send_request) + return mocked_send_request diff --git a/tests/backend_api/test_connector.py b/tests/backend_api/test_connector.py index 5e011fd3..a2de406d 100644 --- a/tests/backend_api/test_connector.py +++ b/tests/backend_api/test_connector.py @@ -1,27 +1,37 @@ import json -from unittest.mock import MagicMock, Mock, call +from unittest.mock import MagicMock, call -from pytest import MonkeyPatch +import pytest +from requests.exceptions import HTTPError from mex.common.backend_api.connector import BackendApiConnector -from mex.common.models import ExtractedPerson +from mex.common.backend_api.models import ExtractedItemsRequest +from mex.common.models import ( + ExtractedPerson, + MergedPerson, + PersonRuleSetRequest, + PersonRuleSetResponse, +) from mex.common.testing import Joker -def test_post_models_mocked( - monkeypatch: MonkeyPatch, extracted_person: ExtractedPerson +@pytest.mark.usefixtures("mocked_backend") +def test_set_authentication_mocked() -> None: + connector = BackendApiConnector.get() + assert connector.session.headers["X-API-Key"] == "dummy_write_key" + + +def test_post_extracted_items_mocked( + mocked_backend: MagicMock, extracted_person: ExtractedPerson ) -> None: - mocked_send_request = MagicMock( - spec=BackendApiConnector._send_request, - return_value=Mock(json=MagicMock(return_value={"identifiers": []})), - ) - monkeypatch.setattr(BackendApiConnector, "_send_request", mocked_send_request) + mocked_return = {"identifiers": [extracted_person.identifier]} + mocked_backend.return_value.json.return_value = mocked_return connector = BackendApiConnector.get() - connector.post_models([extracted_person]) + response = connector.post_extracted_items([extracted_person]) - assert connector.session.headers["X-API-Key"] == "dummy_write_key" - assert mocked_send_request.call_args_list[-1] == call( + assert response.identifiers == [extracted_person.identifier] + assert mocked_backend.call_args == call( "POST", "http://localhost:8080/v0/ingest", None, @@ -32,23 +42,178 @@ def test_post_models_mocked( timeout=10, data=Joker(), ) + assert ( + json.loads(mocked_backend.call_args.kwargs["data"]) + == ExtractedItemsRequest(items=[extracted_person]).model_dump() + ) + + +def test_fetch_extracted_items_mocked( + mocked_backend: MagicMock, extracted_person: ExtractedPerson +) -> None: + mocked_return = {"items": [extracted_person.model_dump()], "total": 3} + mocked_backend.return_value.json.return_value = mocked_return + + connector = BackendApiConnector.get() + response = connector.fetch_extracted_items( + "Tintzmann", + "NGwfzG8ROsrvIiQIVDVy", + entity_type=["ExtractedPerson", "ExtractedContactPoint"], + skip=0, + limit=1, + ) + + assert response.items == [extracted_person] + assert response.total == 3 + + assert mocked_backend.call_args == call( + "GET", + "http://localhost:8080/v0/extracted-item", + { + "q": "Tintzmann", + "stableTargetId": "NGwfzG8ROsrvIiQIVDVy", + "entityType": ["ExtractedPerson", "ExtractedContactPoint"], + "skip": "0", + "limit": "1", + }, + headers={ + "Accept": "application/json", + "User-Agent": "rki/mex", + }, + timeout=10, + ) + + +def test_fetch_merged_items_mocked( + mocked_backend: MagicMock, merged_person: MergedPerson +) -> None: + mocked_return = {"items": [merged_person.model_dump()], "total": 3} + mocked_backend.return_value.json.return_value = mocked_return + + connector = BackendApiConnector.get() + response = connector.fetch_merged_items( + "Tintzmann", + entity_type=["MergedPerson", "MergedContactPoint"], + skip=0, + limit=1, + ) + + assert response.items == [merged_person] + assert response.total == 3 + + assert mocked_backend.call_args == call( + "GET", + "http://localhost:8080/v0/merged-item", + { + "q": "Tintzmann", + "entityType": ["MergedPerson", "MergedContactPoint"], + "skip": "0", + "limit": "1", + }, + headers={ + "Accept": "application/json", + "User-Agent": "rki/mex", + }, + timeout=10, + ) + + +def test_get_merged_item_mocked( + mocked_backend: MagicMock, merged_person: MergedPerson +) -> None: + mocked_return = {"items": [merged_person.model_dump()], "total": 1} + mocked_backend.return_value.json.return_value = mocked_return + + connector = BackendApiConnector.get() + response = connector.get_merged_item("NGwfzG8ROsrvIiQIVDVy") + + assert response == merged_person + + assert mocked_backend.call_args == call( + "GET", + "http://localhost:8080/v0/merged-item", + { + "stableTargetId": "NGwfzG8ROsrvIiQIVDVy", + "limit": "1", + }, + headers={ + "Accept": "application/json", + "User-Agent": "rki/mex", + }, + timeout=10, + ) + + +def test_get_merged_item_error_mocked(mocked_backend: MagicMock) -> None: + mocked_return = {"items": [], "total": 0} + mocked_backend.return_value.json.return_value = mocked_return + + connector = BackendApiConnector.get() + with pytest.raises(HTTPError, match="merged item was not found"): + connector.get_merged_item("NGwfzG8ROsrvIiQIVDVy") + + assert mocked_backend.call_args == call( + "GET", + "http://localhost:8080/v0/merged-item", + { + "stableTargetId": "NGwfzG8ROsrvIiQIVDVy", + "limit": "1", + }, + headers={ + "Accept": "application/json", + "User-Agent": "rki/mex", + }, + timeout=10, + ) + + +def test_preview_merged_item_mocked( + mocked_backend: MagicMock, + merged_person: MergedPerson, + rule_set_request: PersonRuleSetRequest, +) -> None: + mocked_return = merged_person.model_dump() + mocked_backend.return_value.json.return_value = mocked_return + + connector = BackendApiConnector.get() + response = connector.preview_merged_item("NGwfzG8ROsrvIiQIVDVy", rule_set_request) + + assert response == merged_person + + assert mocked_backend.call_args == call( + "GET", + "http://localhost:8080/v0/preview-item/NGwfzG8ROsrvIiQIVDVy", + None, + headers={ + "Accept": "application/json", + "User-Agent": "rki/mex", + }, + timeout=10, + data=Joker(), + ) + assert ( + json.loads(mocked_backend.call_args.kwargs["data"]) + == rule_set_request.model_dump() + ) + - assert json.loads(mocked_send_request.call_args_list[-1].kwargs["data"]) == { - "items": [ - { - "identifier": "e3VhxMhEKyjqN5flzLpiEB", - "hadPrimarySource": "bFQoRhcVH5DHXE", - "identifierInPrimarySource": "00000000-0000-4000-8000-0000000003de", - "stableTargetId": "NGwfzG8ROsrvIiQIVDVy", - "affiliation": ["bFQoRhcVH5DHZg"], - "email": ["TintzmannM@rki.de"], - "familyName": ["Tintzmann"], - "fullName": ["Meinrad I. Tintzmann"], - "givenName": ["Meinrad"], - "isniId": ["https://isni.org/isni/0000000109403744"], - "memberOf": ["bFQoRhcVH5DHV2", "bFQoRhcVH5DHV3"], - "orcidId": ["https://orcid.org/0000-0002-9079-593X"], - "entityType": "ExtractedPerson", - } - ] - } +def test_get_rule_set_mocked( + mocked_backend: MagicMock, rule_set_response: PersonRuleSetResponse +) -> None: + mocked_backend.return_value.json.return_value = rule_set_response.model_dump() + + connector = BackendApiConnector.get() + response = connector.get_rule_set("NGwfzG8ROsrvIiQIVDVy") + + assert response == rule_set_response + + assert mocked_backend.call_args == call( + "GET", + "http://localhost:8080/v0/rule-set/NGwfzG8ROsrvIiQIVDVy", + None, + headers={ + "Accept": "application/json", + "User-Agent": "rki/mex", + }, + timeout=10, + ) diff --git a/tests/conftest.py b/tests/conftest.py index f53ba43f..34573fe3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,15 @@ import pytest -from mex.common.models import ExtractedPerson +from mex.common.models import ( + AdditivePerson, + ExtractedPerson, + MergedPerson, + PersonRuleSetRequest, + PersonRuleSetResponse, + PreventivePerson, + SubtractivePerson, +) from mex.common.types import ( Email, ExtractedPersonIdentifier, @@ -35,3 +43,43 @@ def extracted_person() -> ExtractedPerson: ], orcidId=["https://orcid.org/0000-0002-9079-593X"], ) + + +@pytest.fixture +def merged_person() -> MergedPerson: + """Return a dummy merged person for testing purposes.""" + return MergedPerson.model_construct( + identifier=MergedPersonIdentifier.generate(seed=876), + affiliation=[MergedOrganizationIdentifier.generate(seed=300)], + email=[Email("TintzmannM@rki.de")], + familyName=["Tintzmann"], + givenName=["Meinrad"], + fullName=["Meinrad I. Tintzmann"], + isniId=["https://isni.org/isni/0000000109403744"], + memberOf=[ + MergedOrganizationalUnitIdentifier.generate(seed=100), + MergedOrganizationalUnitIdentifier.generate(seed=101), + ], + orcidId=["https://orcid.org/0000-0002-9079-593X"], + ) + + +@pytest.fixture() +def rule_set_request() -> PersonRuleSetRequest: + """Return a dummy person rule set request for testing purposes.""" + return PersonRuleSetRequest( + additive=AdditivePerson(), + subtractive=SubtractivePerson(fullName="That's not my name!"), + preventive=PreventivePerson(), + ) + + +@pytest.fixture() +def rule_set_response() -> PersonRuleSetResponse: + """Return a dummy person rule set response for testing purposes.""" + return PersonRuleSetResponse( + stableTargetId=MergedPersonIdentifier.generate(seed=876), + additive=AdditivePerson(), + subtractive=SubtractivePerson(fullName="That's not my name!"), + preventive=PreventivePerson(), + ) diff --git a/tests/connector/test_http.py b/tests/connector/test_http.py index de389ad8..6abc94cd 100644 --- a/tests/connector/test_http.py +++ b/tests/connector/test_http.py @@ -6,10 +6,7 @@ from pytest import MonkeyPatch from requests import JSONDecodeError, Response -from mex.common.connector import ( - CONNECTOR_STORE, - HTTPConnector, -) +from mex.common.connector import CONNECTOR_STORE, HTTPConnector class DummyHTTPConnector(HTTPConnector): @@ -44,7 +41,10 @@ def test_init_mocked(mocked_dummy_session: MagicMock) -> None: "https://www.example.com/_system/check", None, timeout=10, - headers={"Accept": "application/json"}, + headers={ + "Accept": "application/json", + "User-Agent": "rki/mex", + }, ) @@ -66,13 +66,23 @@ def test_reset_all_connectors(mocked_dummy_session: MagicMock) -> None: {}, MagicMock(status_code=204, json=MagicMock(side_effect=JSONDecodeError)), {}, - {"headers": {"Accept": "application/json"}}, + { + "headers": { + "Accept": "application/json", + "User-Agent": "rki/mex", + }, + }, ), ( {}, MagicMock(status_code=200, json=MagicMock(return_value={"foo": "bar"})), {"foo": "bar"}, - {"headers": {"Accept": "application/json"}}, + { + "headers": { + "Accept": "application/json", + "User-Agent": "rki/mex", + }, + }, ), ( {"q": "SELECT status;"}, diff --git a/tests/sinks/test_backend_api.py b/tests/sinks/test_backend_api.py index b8756b27..52ab29f1 100644 --- a/tests/sinks/test_backend_api.py +++ b/tests/sinks/test_backend_api.py @@ -1,9 +1,9 @@ from unittest.mock import MagicMock, Mock -from uuid import UUID from pytest import MonkeyPatch from mex.common.backend_api.connector import BackendApiConnector +from mex.common.backend_api.models import IdentifiersResponse from mex.common.models import ExtractedPerson from mex.common.sinks.backend_api import post_to_backend_api @@ -16,10 +16,12 @@ def __init__(self: BackendApiConnector) -> None: monkeypatch.setattr(BackendApiConnector, "__init__", __init__) - response = [UUID("00000000-0000-4000-8000-000000339191")] - post_models = Mock(return_value=response) - monkeypatch.setattr(BackendApiConnector, "post_models", post_models) + response = IdentifiersResponse(identifiers=[extracted_person.identifier]) + post_extracted_items = Mock(return_value=response) + monkeypatch.setattr( + BackendApiConnector, "post_extracted_items", post_extracted_items + ) model_ids = list(post_to_backend_api([extracted_person])) - assert model_ids == response - post_models.assert_called_once_with([extracted_person]) + assert model_ids == response.identifiers + post_extracted_items.assert_called_once_with([extracted_person])