From a5c818e8a2144209d83ce5f09eb785a7d632dedb Mon Sep 17 00:00:00 2001 From: Paul Schilling Date: Fri, 15 Nov 2024 12:15:08 +0100 Subject: [PATCH] [#2871] Refactor ZGW import logic - overwrite only editable fields when importing ZGW entities, skip read-only fields (url, domein, rsin) --- src/open_inwoner/openzaak/import_export.py | 259 +++++++++++---- .../openzaak/tests/test_import_export.py | 311 ++++++++---------- 2 files changed, 323 insertions(+), 247 deletions(-) diff --git a/src/open_inwoner/openzaak/import_export.py b/src/open_inwoner/openzaak/import_export.py index f2318ecf15..ebdea8c11e 100644 --- a/src/open_inwoner/openzaak/import_export.py +++ b/src/open_inwoner/openzaak/import_export.py @@ -8,6 +8,7 @@ from django.core import serializers from django.core.files.storage import Storage +from django.core.serializers.base import DeserializationError from django.db import transaction from django.db.models import QuerySet @@ -22,6 +23,146 @@ logger = logging.getLogger(__name__) +class ZGWImportError(Exception): + pass + + +def check_catalogus_config_exists(source_config): + try: + CatalogusConfig.objects.get_by_natural_key( + domein=source_config.domein, rsin=source_config.rsin + ) + except CatalogusConfig.MultipleObjectsReturned: + raise ZGWImportError( + "Got multiple results for CatalogusConfig with domain={domein} and rsin={rsin}".format( + domein=source_config.domein, + rsin=source_config.rsin, + ) + ) + except CatalogusConfig.DoesNotExist: + raise ZGWImportError( + "CatalogusConfig not found in target environment: domein={domein} and rsin={rsin}".format( + domein=source_config.domein, rsin=source_config.rsin + ), + ) + + +def update_zaaktype_config(source_config): + catalogus_domein = source_config.catalogus.domein + catalogus_rsin = source_config.catalogus.rsin + + try: + target = ZaakTypeConfig.objects.get_by_natural_key( + identificatie=source_config.identificatie, + catalogus_domein=catalogus_domein, + catalogus_rsin=source_config.catalogus.rsin, + ) + except ZaakTypeConfig.MultipleObjectsReturned: + raise ZGWImportError( + "Got multiple results for ZaakTypeConfig with identificatie={identificatie}, " + "catalogus domein={domein} and catalogus_rsin={rsin}".format( + identificatie=source_config.identificatie, + domein=catalogus_domein, + rsin=catalogus_rsin, + ) + ) + except (CatalogusConfig.DoesNotExist, ZaakTypeConfig.DoesNotExist): + raise ZGWImportError( + "ZaakTypeConfig not found in target environment: identificatie={identificatie}, " + "catalogus domein={domein}, catalogus rsin={rsin}".format( + identificatie=source_config.identificatie, + domein=source_config.domein, + rsin=source_config.rsin, + ), + ) + else: + update_fields = [ + "notify_status_changes", + "external_document_upload_url", + "document_upload_enabled", + "contact_form_enabled", + "contact_subject_code", + "relevante_zaakperiode", + ] + for field in update_fields: + val = getattr(source_config, field, None) + setattr(target, field, val) + target.save() + + +def _update_nested_zgw_config(source_config: type, update_fields: list[str]): + zaaktype_config_identificatie = source_config.zaaktype_config.identificatie + catalogus_domein = source_config.zaaktype_config.catalogus.domein + catalogus_rsin = source_config.zaaktype_config.catalogus.rsin + + try: + target = source_config.__class__.objects.get_by_natural_key( + omschrijving=source_config.omschrijving, + zaak_type_config_identificatie=zaaktype_config_identificatie, + catalogus_domein=catalogus_domein, + catalogus_rsin=catalogus_rsin, + ) + except ZaakTypeInformatieObjectTypeConfig.MultipleObjectsReturned: + raise ZGWImportError( + f"Got multiple results for {source_config.__class__.__name__} with: " + "zaaktype config={zaaktype_config_identificatie}, catalogus_domein={catalogus_domein}, " + "catalogus_rsin={catalogus_rsin}".format( + zaaktype_config_identificatie=zaaktype_config_identificatie, + catalogus_domein=catalogus_domein, + catalogus_rsin=catalogus_rsin, + ), + ) + except ZaakTypeInformatieObjectTypeConfig.DoesNotExist: + raise ZGWImportError( + f"{source_config.__class__.__name__} not found in target environment: " + "omschrijving={omschrijving}, catalogus domein={domein}, catalogus rsin={rsin}".format( + omschrijving=source_config.omschrijving, + domein=catalogus_domein, + rsin=catalogus_rsin, + ), + ) + else: + for field in update_fields: + val = getattr(source_config, field, None) + setattr(target, field, val) + target.save() + + +def update_zaaktype_informatie_objecttype_config(source_config): + update_fields = [ + "zaaktype_uuids", + "document_upload_enabled", + "document_notification_enabled", + ] + _update_nested_zgw_config(source_config, update_fields) + + +def update_zaaktype_statustype_config(source_config): + update_fields = [ + "statustekst", + "zaaktype_uuids", + "status_indicator", + "status_indicator_text", + "document_upload_description", + "desciption", + "notify_status_change", + "action_required", + "document_upload_enabled", + "call_to_action_url", + "call_to_action_text", + "case_link_text", + ] + _update_nested_zgw_config(source_config, update_fields) + + +def update_zaaktype_resultaattype_config(source_config): + update_fields = [ + "zaaktype_uuids", + "description", + ] + _update_nested_zgw_config(source_config, update_fields) + + @dataclasses.dataclass(frozen=True) class CatalogusConfigExport: """Gather and export CatalogusConfig(s) and all associated relations.""" @@ -113,9 +254,10 @@ class CatalogusConfigImport: total_rows_processed: int = 0 catalogus_configs_imported: int = 0 zaaktype_configs_imported: int = 0 - zaak_inormatie_object_type_configs_imported: int = 0 + zaak_informatie_object_type_configs_imported: int = 0 zaak_status_type_configs_imported: int = 0 zaak_resultaat_type_configs_imported: int = 0 + import_errors: list | None = None @staticmethod def _get_url_root(url: str) -> str: @@ -149,90 +291,65 @@ def _lines_iter_from_jsonl_stream_or_string( # Reset the stream in case it gets re-used lines.seek(0) - @classmethod - def _rewrite_jsonl_url_references( - cls, stream_or_string: IO | str - ) -> Generator[str, Any, None]: - # The assumption is that the exporting and importing instance both have - # a `Service` with the same slug as the `Service` referenced in the - # `configued_from` attribute of the imported CatalogusConfig. The - # assumption is further that all URLs in the imported objects are - # prefixed by an URL that matches the API root in the service. Because - # of this, the import file will contain URLs with a base URL pointing to - # the `api_root`` of the `configured_from` Service on the _source_ - # instance, and has to be re-written to match the `api_root` of the - # `configured_from` Service on the _target_ instance. Put differently, - # we assume that we are migrating ZGW objects that _do not differ_ as - # far as the ZGW objects themselves are concerned (apart from the URL, - # they essentially point to the same ZGW backend), but that they _do_ - # differ in terms of additional model fields that do not have their - # source of truth in the ZGW backends. - # - # This expectation is also encoded in our API clients: you can only - # fetch ZGW objects using the ApePie clients if the root of those - # objects matches the configured API root. - - base_url_mapping = {} - for deserialized_object in serializers.deserialize( - "jsonl", - filter( - lambda row: ('"model": "openzaak.catalogusconfig"' in row), - cls._lines_iter_from_jsonl_stream_or_string(stream_or_string), - ), - use_natural_foreign_keys=True, - use_natural_primary_keys=True, - ): - object_type: str = deserialized_object.object.__class__.__name__ - - if object_type == "CatalogusConfig": - target_base_url = cls._get_url_root( - deserialized_object.object.service.api_root - ) - source_base_url = cls._get_url_root(deserialized_object.object.url) - base_url_mapping[source_base_url] = target_base_url - else: - # https://www.xkcd.com/2200/ - logger.error( - "Tried to filter for catalogus config objects, but also got: %s", - object_type, - ) - - for line in cls._lines_iter_from_jsonl_stream_or_string(stream_or_string): - source_url_found = False - for source, target in base_url_mapping.items(): - line = line.replace(source, target) - source_url_found = True - - if not source_url_found: - raise ValueError("Unable to rewrite ZGW urls") - - yield line - @classmethod @transaction.atomic() def from_jsonl_stream_or_string(cls, stream_or_string: IO | str) -> Self: model_to_counter_mapping = { "CatalogusConfig": "catalogus_configs_imported", "ZaakTypeConfig": "zaaktype_configs_imported", - "ZaakTypeInformatieObjectTypeConfig": "zaak_inormatie_object_type_configs_imported", + "ZaakTypeInformatieObjectTypeConfig": "zaak_informatie_object_type_configs_imported", "ZaakTypeStatusTypeConfig": "zaak_status_type_configs_imported", "ZaakTypeResultaatTypeConfig": "zaak_resultaat_type_configs_imported", } - object_type_counts = defaultdict(int) - for deserialized_object in serializers.deserialize( - "jsonl", - cls._rewrite_jsonl_url_references(stream_or_string), - use_natural_foreign_keys=True, - use_natural_primary_keys=True, - ): - deserialized_object.save() - object_type = deserialized_object.object.__class__.__name__ - object_type_counts[object_type] += 1 + import_errors = [] + for line in cls._lines_iter_from_jsonl_stream_or_string(stream_or_string): + try: + (deserialized_object,) = serializers.deserialize( + "jsonl", + line, + use_natural_primary_keys=True, + use_natural_foreign_keys=True, + ) + except DeserializationError as exc: + exc_source = type(exc.__context__) + if ( + exc_source is CatalogusConfig.DoesNotExist + or ZaakTypeConfig.DoesNotExist + ): + logger.error(exc) + import_errors.append(exc) + else: + source_config = deserialized_object.object + try: + match source_config: + case CatalogusConfig(): + check_catalogus_config_exists(source_config=source_config) + case ZaakTypeConfig(): + update_zaaktype_config(source_config=source_config) + case ZaakTypeInformatieObjectTypeConfig(): + update_zaaktype_informatie_objecttype_config( + source_config=source_config + ) + case ZaakTypeStatusTypeConfig(): + update_zaaktype_statustype_config( + source_config=source_config + ) + case ZaakTypeResultaatTypeConfig(): + update_zaaktype_resultaattype_config( + source_config=source_config + ) + except ZGWImportError as exc: + logger.error(exc) + import_errors.append(exc) + else: + object_type = deserialized_object.object.__class__.__name__ + object_type_counts[object_type] += 1 creation_kwargs = { "total_rows_processed": sum(object_type_counts.values()), + "import_errors": import_errors, } for model_name, counter_field in model_to_counter_mapping.items(): diff --git a/src/open_inwoner/openzaak/tests/test_import_export.py b/src/open_inwoner/openzaak/tests/test_import_export.py index 55bb2959d2..414cb456ee 100644 --- a/src/open_inwoner/openzaak/tests/test_import_export.py +++ b/src/open_inwoner/openzaak/tests/test_import_export.py @@ -1,6 +1,7 @@ import io from django.core.files.storage.memory import InMemoryStorage +from django.core.serializers.base import DeserializationError from django.test import TestCase from open_inwoner.openzaak.import_export import ( @@ -270,235 +271,193 @@ def setUp(self): self.json_lines = [ '{"model": "openzaak.catalogusconfig", "fields": {"url": "https://foo.0.maykinmedia.nl", "domein": "DM-0", "rsin": "123456789", "service": ["service-0"]}}', - '{"model": "openzaak.catalogusconfig", "fields": {"url": "https://foo.1.maykinmedia.nl", "domein": "DM-1", "rsin": "123456789", "service": ["service-1"]}}', '{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"https://foo.0.maykinmedia.nl\\"]", "catalogus": ["DM-0", "123456789"], "identificatie": "ztc-id-a-0", "omschrijving": "zaaktypeconfig", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": false, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', - '{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"https://foo.1.maykinmedia.nl\\"]", "catalogus": ["DM-1", "123456789"], "identificatie": "ztc-id-a-1", "omschrijving": "zaaktypeconfig", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": false, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', - '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "informatieobjecttype_url": "http://foo.0.maykinmedia.nl", "omschrijving": "informatieobject", "zaaktype_uuids": "[]", "document_upload_enabled": false, "document_notification_enabled": false}}', - '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-1", "DM-1", "123456789"], "informatieobjecttype_url": "http://foo.1.maykinmedia.nl", "omschrijving": "informatieobject", "zaaktype_uuids": "[]", "document_upload_enabled": false, "document_notification_enabled": false}}', - '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "statustype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', - '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-1", "DM-1", "123456789"], "statustype_url": "https://foo.1.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', - '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "resultaattype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "resultaat", "zaaktype_uuids": "[]", "description": ""}}', - '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-1", "DM-1", "123456789"], "resultaattype_url": "https://foo.1.maykinmedia.nl", "omschrijving": "resultaat", "zaaktype_uuids": "[]", "description": ""}}', + '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "informatieobjecttype_url": "http://foo.0.maykinmedia.nl", "omschrijving": "informatieobject", "zaaktype_uuids": "[]", "document_upload_enabled": true, "document_notification_enabled": true}}', + '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "statustype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "statustekst nieuw", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', + '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "resultaattype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "resultaat", "zaaktype_uuids": "[]", "description": "description new"}}', ] - self.jsonl = "\n".join(self.json_lines) - - def test_import_jsonl_creates_objects(self): - self.storage.save("import.jsonl", io.StringIO(self.jsonl)) - - import_result = CatalogusConfigImport.import_from_jsonl_file_in_django_storage( - "import.jsonl", self.storage - ) - self.assertEqual( - import_result, - CatalogusConfigImport( - total_rows_processed=10, - catalogus_configs_imported=2, - zaaktype_configs_imported=2, - zaak_inormatie_object_type_configs_imported=2, - zaak_status_type_configs_imported=2, - zaak_resultaat_type_configs_imported=2, - ), - ) - - self.assertEqual(CatalogusConfig.objects.count(), 2) - self.assertEqual(ZaakTypeConfig.objects.count(), 2) - self.assertEqual(ZaakTypeInformatieObjectTypeConfig.objects.count(), 2) - self.assertEqual(ZaakTypeStatusTypeConfig.objects.count(), 2) - self.assertEqual(ZaakTypeResultaatTypeConfig.objects.count(), 2) - - def test_import_jsonl_merges_objects(self): - CatalogusConfigFactory( - url="https://foo.0.maykinmedia.nl", - domein="FOO", - rsin="123456789", - service=self.service, - ) - merge_line = '{"model": "openzaak.catalogusconfig", "fields": {"url": "https://foo.0.maykinmedia.nl", "domein": "BAR", "rsin": "987654321", "service": ["service-0"]}}' - - import_result = CatalogusConfigImport.from_jsonl_stream_or_string(merge_line) - - self.assertEqual(import_result.catalogus_configs_imported, 1) - self.assertEqual(import_result.total_rows_processed, 1) - - self.assertEqual( - list(CatalogusConfig.objects.values_list("url", "domein", "rsin")), - [("https://foo.0.maykinmedia.nl", "BAR", "987654321")], - msg="Value of sole CatalogusConfig matches imported values, not original values", - ) - - def test_bad_import_types(self): - for bad_type in (set(), list(), b""): - with self.assertRaises(ValueError): - CatalogusConfigImport.from_jsonl_stream_or_string(bad_type) - - def test_valid_input_types_are_accepted(self): - for input in ( - io.StringIO(self.jsonl), - io.BytesIO(self.jsonl.encode("utf-8")), - self.jsonl, - ): - with self.subTest(f"Input type {type(input)}"): - import_result = CatalogusConfigImport.from_jsonl_stream_or_string(input) - self.assertEqual( - import_result, - CatalogusConfigImport( - total_rows_processed=10, - catalogus_configs_imported=2, - zaaktype_configs_imported=2, - zaak_inormatie_object_type_configs_imported=2, - zaak_status_type_configs_imported=2, - zaak_resultaat_type_configs_imported=2, - ), - ) - - self.assertEqual(CatalogusConfig.objects.count(), 2) - self.assertEqual(ZaakTypeConfig.objects.count(), 2) - self.assertEqual(ZaakTypeInformatieObjectTypeConfig.objects.count(), 2) - self.assertEqual(ZaakTypeStatusTypeConfig.objects.count(), 2) - self.assertEqual(ZaakTypeResultaatTypeConfig.objects.count(), 2) - - def test_import_is_atomic(self): - bad_line = '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {}}\n' - bad_jsonl = self.jsonl + "\n" + bad_line - - try: - CatalogusConfigImport.from_jsonl_stream_or_string( - stream_or_string=bad_jsonl - ) - except Exception: - pass - - counts = ( - CatalogusConfig.objects.count(), - ZaakTypeConfig.objects.count(), - ZaakTypeInformatieObjectTypeConfig.objects.count(), - ZaakTypeStatusTypeConfig.objects.count(), - ZaakTypeResultaatTypeConfig.objects.count(), - ) - expected_counts = (0, 0, 0, 0, 0) - - self.assertEqual( - counts, - expected_counts, - msg="Import should have merged, and not created new values", - ) - - -class RewriteUrlsImportTests(TestCase): - def setUp(self): - self.service = ServiceFactory( - slug="constant-api-slug", api_root="http://one.maykinmedia.nl" - ) - - import_lines = [ - '{"model": "openzaak.catalogusconfig", "fields": {"url": "http://one.maykinmedia.nl/catalogus/1", "domein": "ALLE", "rsin": "1234568", "service": ["constant-api-slug"]}}', - '{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"http://one.maykinmedia.nl/types/1\\", \\"http://one.maykinmedia.nl/types/2\\"]", "catalogus": ["http://one.maykinmedia.nl/catalogus/1"], "identificatie": "zt-1", "omschrijving": "iGsHCEkCpEJyDLeAaytskGiAXSAPVVthCvOdbNdpZZcCciXFnZGltXFYsYigSkIZiaqMEvSPftMgIYyW", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": false, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', - '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["zt-1", "http://one.maykinmedia.nl/catalogus/1"], "informatieobjecttype_url": "http://one.maykinmedia.nl/iotype/1", "omschrijving": "IzNqfWpVpbyMEjSXTqQUlslqAUYFdILFlSDAelAkfTROWptqgIRCmaIoWCBMBAozsJLWxGoJqmBLPCHy", "zaaktype_uuids": "[]", "document_upload_enabled": false, "document_notification_enabled": false}}', - '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["zt-1", "http://one.maykinmedia.nl/catalogus/1"], "statustype_url": "http://one.maykinmedia.nl/status-type/1", "omschrijving": "BHEJLQkSTdMPGtSzgnIbIdhMvFiNOBHmFQkRvLxHUkmafelprqCpcuAZzqMWBLgqNkGmXpzWPjhWqKjk", "statustekst": "", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', - '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["zt-1", "http://one.maykinmedia.nl/catalogus/1"], "resultaattype_url": "http://one.maykinmedia.nl/resultaat-type/1", "omschrijving": "", "zaaktype_uuids": "[]", "description": ""}}', + self.json_lines_bogus = [ + '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["bogus", "DM-1", "666666666"], "statustype_url": "https://foo.1.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', ] - self.jsonl = "\n".join(import_lines) + self.jsonl = "\n".join(self.json_lines) + self.jsonl_extra = "\n".join(self.json_lines + self.json_lines_bogus) - def _create_fixtures(self, base_url: str): + def create_fixtures(self, base_url: str): catalogus = CatalogusConfigFactory( url=f"{base_url}/catalogus/1", service=self.service, - domein="ALLE", - rsin="1234568", + domein="DM-0", + rsin="123456789", ) zt = ZaakTypeConfigFactory( catalogus=catalogus, - identificatie="zt-1", + identificatie="ztc-id-a-0", + omschrijving="zaaktypeconfig", urls=[ f"{base_url}/types/1", f"{base_url}/types/2", ], ) ZaakTypeInformatieObjectTypeConfigFactory( - zaaktype_config=zt, informatieobjecttype_url=f"{base_url}/iotype/1", + zaaktype_config=zt, + omschrijving="informatieobject", + document_upload_enabled=False, + document_notification_enabled=False, ) ZaakTypeStatusTypeConfigFactory( - zaaktype_config=zt, statustype_url=f"{base_url}/status-type/1" + statustype_url=f"{base_url}/status-type/1", + zaaktype_config=zt, + omschrijving="status omschrijving", + statustekst="statustekst oud", + notify_status_change=False, ) ZaakTypeResultaatTypeConfigFactory( - zaaktype_config=zt, resultaattype_url=f"{base_url}/resultaat-type/1", + zaaktype_config=zt, + omschrijving="resultaat", + description="description old", ) - def test_jsonl_url_rewrite(self): - self.service.api_root = "http://two.maykinmedia.nl" - self.service.save() - - rewritten_lines = list( - CatalogusConfigImport._rewrite_jsonl_url_references(self.jsonl) - ) - expected_lines = [ - '{"model": "openzaak.catalogusconfig", "fields": {"url": "http://two.maykinmedia.nl/catalogus/1", "domein": "ALLE", "rsin": "1234568", "service": ["constant-api-slug"]}}', - '{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"http://two.maykinmedia.nl/types/1\\", \\"http://two.maykinmedia.nl/types/2\\"]", "catalogus": ["http://two.maykinmedia.nl/catalogus/1"], "identificatie": "zt-1", "omschrijving": "iGsHCEkCpEJyDLeAaytskGiAXSAPVVthCvOdbNdpZZcCciXFnZGltXFYsYigSkIZiaqMEvSPftMgIYyW", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": false, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', - '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["zt-1", "http://two.maykinmedia.nl/catalogus/1"], "informatieobjecttype_url": "http://two.maykinmedia.nl/iotype/1", "omschrijving": "IzNqfWpVpbyMEjSXTqQUlslqAUYFdILFlSDAelAkfTROWptqgIRCmaIoWCBMBAozsJLWxGoJqmBLPCHy", "zaaktype_uuids": "[]", "document_upload_enabled": false, "document_notification_enabled": false}}', - '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["zt-1", "http://two.maykinmedia.nl/catalogus/1"], "statustype_url": "http://two.maykinmedia.nl/status-type/1", "omschrijving": "BHEJLQkSTdMPGtSzgnIbIdhMvFiNOBHmFQkRvLxHUkmafelprqCpcuAZzqMWBLgqNkGmXpzWPjhWqKjk", "statustekst": "", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', - '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["zt-1", "http://two.maykinmedia.nl/catalogus/1"], "resultaattype_url": "http://two.maykinmedia.nl/resultaat-type/1", "omschrijving": "", "zaaktype_uuids": "[]", "description": ""}}', - ] + def test_import_jsonl_update_success(self): + self.create_fixtures(base_url="https://foo.0.maykinmedia.nl") + self.storage.save("import.jsonl", io.StringIO(self.jsonl)) - self.assertEqual( - rewritten_lines, - expected_lines, - msg="All URLs should be rewritten to match the target service root", + import_result = CatalogusConfigImport.import_from_jsonl_file_in_django_storage( + "import.jsonl", self.storage ) - def test_rewrite_target_diverges_from_existing_objects(self): - self._create_fixtures("http://one.maykinmedia.nl/") - self.service.api_root = "http://two.maykinmedia.nl" - self.service.save() - - import_result = CatalogusConfigImport.from_jsonl_stream_or_string(self.jsonl) - + # check import self.assertEqual( import_result, CatalogusConfigImport( total_rows_processed=5, catalogus_configs_imported=1, zaaktype_configs_imported=1, - zaak_inormatie_object_type_configs_imported=1, + zaak_informatie_object_type_configs_imported=1, zaak_status_type_configs_imported=1, zaak_resultaat_type_configs_imported=1, + import_errors=[], ), ) - counts = ( - CatalogusConfig.objects.count(), - ZaakTypeConfig.objects.count(), - ZaakTypeInformatieObjectTypeConfig.objects.count(), - ZaakTypeStatusTypeConfig.objects.count(), - ZaakTypeResultaatTypeConfig.objects.count(), - ) - expected_counts = (2, 2, 2, 2, 2) + # check number of configs + self.assertEqual(CatalogusConfig.objects.count(), 1) + self.assertEqual(ZaakTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeInformatieObjectTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeStatusTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeResultaatTypeConfig.objects.count(), 1) + + # check content of configs + zaaktype_statustype_config = ZaakTypeStatusTypeConfig.objects.get() + self.assertEqual(zaaktype_statustype_config.statustekst, "statustekst nieuw") + self.assertEqual(zaaktype_statustype_config.notify_status_change, True) + + zaaktype_resultaattype_config = ZaakTypeResultaatTypeConfig.objects.get() + self.assertEqual(zaaktype_resultaattype_config.description, "description new") + zaaktype_informatie_objecttype_config = ( + ZaakTypeInformatieObjectTypeConfig.objects.get() + ) self.assertEqual( - counts, - expected_counts, - msg="Import should have merged, and not created new values", + zaaktype_informatie_objecttype_config.document_upload_enabled, True + ) + self.assertEqual( + zaaktype_informatie_objecttype_config.document_notification_enabled, True + ) + + def test_import_jsonl_update_skips_missing(self): + self.create_fixtures(base_url="https://foo.0.maykinmedia.nl") + self.storage.save("import.jsonl", io.StringIO(self.jsonl_extra)) + + import_result = CatalogusConfigImport.import_from_jsonl_file_in_django_storage( + "import.jsonl", self.storage ) - def test_rewrite_target_matches_from_existing_objects(self): - self.service.api_root = "http://two.maykinmedia.nl" - self.service.save() - self._create_fixtures("http://two.maykinmedia.nl") + error = DeserializationError( + "ZaakTypeConfig matching query does not exist.: (openzaak.zaaktypestatustypeconfig:pk=None) field_value was '['bogus', 'DM-1', '666666666']'" + ) + import_error = import_result.import_errors[0] - import_result = CatalogusConfigImport.from_jsonl_stream_or_string(self.jsonl) + # check import self.assertEqual( import_result, CatalogusConfigImport( total_rows_processed=5, catalogus_configs_imported=1, zaaktype_configs_imported=1, - zaak_inormatie_object_type_configs_imported=1, + zaak_informatie_object_type_configs_imported=1, zaak_status_type_configs_imported=1, zaak_resultaat_type_configs_imported=1, + import_errors=[error], ), ) + # check number of configs + self.assertEqual(CatalogusConfig.objects.count(), 1) + self.assertEqual(ZaakTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeInformatieObjectTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeStatusTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeResultaatTypeConfig.objects.count(), 1) + + def test_import_jsonl_fails_with_catalogus_domein_rsin_mismatch(self): + CatalogusConfigFactory( + url="https://foo.0.maykinmedia.nl", + domein="FOO", + rsin="123456789", + service=self.service, + ) + import_lines = [ + '{"model": "openzaak.catalogusconfig", "fields": {"url": "https://foo.0.maykinmedia.nl", "domein": "BAR", "rsin": "987654321", "service": ["service-0"]}}', + '{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"https://foo.0.maykinmedia.nl\\"]", "catalogus": ["DM-0", "123456789"], "identificatie": "ztc-id-a-0", "omschrijving": "zaaktypeconfig", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": false, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', + ] + import_line = "\n".join(import_lines) + + with self.assertLogs( + logger="open_inwoner.openzaak.import_export", level="ERROR" + ) as cm: + import_result = CatalogusConfigImport.from_jsonl_stream_or_string( + import_line + ) + self.assertEqual( + cm.output, + [ + # error from trying to load existing CatalogusConfig + "ERROR:open_inwoner.openzaak.import_export:" + "CatalogusConfig not found in target environment: domein=BAR and rsin=987654321", + # error from deserializing nested ZGW objects + "ERROR:open_inwoner.openzaak.import_export:CatalogusConfig matching query does not " + "exist.: (openzaak.zaaktypeconfig:pk=None) field_value was '['DM-0', '123456789']'", + ], + ) + + self.assertEqual(CatalogusConfig.objects.count(), 1) + + self.assertEqual(import_result.catalogus_configs_imported, 0) + self.assertEqual(import_result.total_rows_processed, 0) + + self.assertEqual( + list(CatalogusConfig.objects.values_list("url", "domein", "rsin")), + [("https://foo.0.maykinmedia.nl", "FOO", "123456789")], + msg="Value of sole CatalogusConfig matches imported values, not original values", + ) + + def test_bad_import_types(self): + for bad_type in (set(), list(), b""): + with self.assertRaises(ValueError): + CatalogusConfigImport.from_jsonl_stream_or_string(bad_type) + + def test_import_is_atomic(self): + bad_line = '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {}}\n' + bad_jsonl = self.jsonl + "\n" + bad_line + + try: + CatalogusConfigImport.from_jsonl_stream_or_string( + stream_or_string=bad_jsonl + ) + except Exception: + pass + counts = ( CatalogusConfig.objects.count(), ZaakTypeConfig.objects.count(), @@ -506,7 +465,7 @@ def test_rewrite_target_matches_from_existing_objects(self): ZaakTypeStatusTypeConfig.objects.count(), ZaakTypeResultaatTypeConfig.objects.count(), ) - expected_counts = (1, 1, 1, 1, 1) + expected_counts = (0, 0, 0, 0, 0) self.assertEqual( counts,