Merge pull request #1503 from maykinmedia/task/2903-import-export-zaa…

…ktype-config

[#2903] Refactor import/export for ZaakType configs

src/open_inwoner/openzaak/admin.py

@@ -13,15 +13,11 @@
 from django.utils.html import format_html, format_html_join
 from django.utils.translation import gettext_lazy as _, ngettext
 
-from import_export.admin import ImportExportMixin
 from privates.storages import PrivateMediaFileSystemStorage
 from solo.admin import SingletonModelAdmin
 
 from open_inwoner.ckeditor5.widgets import CKEditorWidget
-from open_inwoner.openzaak.import_export import (
-    CatalogusConfigExport,
-    CatalogusConfigImport,
-)
+from open_inwoner.openzaak.import_export import ZGWConfigExport, ZGWConfigImport
 from open_inwoner.utils.forms import LimitedUploadFileField
 
 from .models import (
@@ -137,14 +133,14 @@ def get_urls(self):
             path(
                 "import-catalogus-dump/",
                 self.admin_site.admin_view(self.process_file_view),
-                name="upload_zgw_import_file",
+                name="upload_catalogus_import_file",
             ),
         ]
         return custom_urls + urls
 
     @admin.action(description=_("Export to file"))
     def export_catalogus_configs(modeladmin, request, queryset):
-        export = CatalogusConfigExport.from_catalogus_configs(queryset)
+        export = ZGWConfigExport.from_catalogus_configs(queryset)
         response = StreamingHttpResponse(
             export.as_jsonl_iter(),
             content_type="application/json",
@@ -167,7 +163,7 @@ def process_file_view(self, request):
 
                 try:
                     import_result = (
-                        CatalogusConfigImport.import_from_jsonl_file_in_django_storage(
+                        ZGWConfigImport.import_from_jsonl_file_in_django_storage(
                             target_file_name,
                             storage,
                         )
@@ -374,7 +370,8 @@ def has_delete_permission(self, request, obj=None):
 
 
 @admin.register(ZaakTypeConfig)
-class ZaakTypeConfigAdmin(ImportExportMixin, admin.ModelAdmin):
+class ZaakTypeConfigAdmin(admin.ModelAdmin):
+    change_list_template = "admin/zaaktypeconfig_change_list.html"
     inlines = [
         ZaakTypeInformatieObjectTypeConfigInline,
         ZaakTypeStatusTypeConfigInline,
@@ -383,6 +380,7 @@ class ZaakTypeConfigAdmin(ImportExportMixin, admin.ModelAdmin):
     actions = [
         "mark_as_notify_status_changes",
         "mark_as_not_notify_status_changes",
+        "export_zaaktype_configs",
     ]
     fields = [
         "urls",
@@ -437,6 +435,94 @@ class ZaakTypeConfigAdmin(ImportExportMixin, admin.ModelAdmin):
     ]
     ordering = ("identificatie", "catalogus__domein")
 
+    def get_urls(self):
+        urls = super().get_urls()
+        custom_urls = [
+            path(
+                "import-zaaktype-dump/",
+                self.admin_site.admin_view(self.process_file_view),
+                name="upload_zaaktype_import_file",
+            ),
+        ]
+        return custom_urls + urls
+
+    @admin.action(description=_("Export to file"))
+    def export_zaaktype_configs(modeladmin, request, queryset):
+        export = ZGWConfigExport.from_zaaktype_configs(queryset)
+        response = StreamingHttpResponse(
+            export.as_jsonl_iter(),
+            content_type="application/json",
+        )
+        response[
+            "Content-Disposition"
+        ] = 'attachment; filename="zgw-zaaktype-export.json"'
+        return response
+
+    def process_file_view(self, request):
+        form = ImportZGWExportFileForm()
+
+        if request.method == "POST":
+            form = ImportZGWExportFileForm(request.POST, request.FILES)
+            if form.is_valid():
+                storage = PrivateMediaFileSystemStorage()
+                timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+                target_file_name = f"zgw_import_dump_{timestamp}.json"
+                storage.save(target_file_name, request.FILES["zgw_export_file"])
+
+                try:
+                    import_result = (
+                        ZGWConfigImport.import_from_jsonl_file_in_django_storage(
+                            target_file_name,
+                            storage,
+                        )
+                    )
+                    self.message_user(
+                        request,
+                        _(
+                            "%(num_rows)d item(s) processed in total, with %(error_rows)d failing row(s)."
+                            % {
+                                "num_rows": import_result.total_rows_processed,
+                                "error_rows": len(import_result.import_errors),
+                            }
+                        ),
+                        messages.SUCCESS
+                        if not import_result.import_errors
+                        else messages.WARNING,
+                    )
+                    if errors := import_result.import_errors:
+                        msgs_deduped = set(error.__str__() for error in errors)
+                        error_msg_iterator = ([msg] for msg in msgs_deduped)
+
+                        error_msg_html = format_html_join(
+                            "\n", "<p> - {}</p>", error_msg_iterator
+                        )
+                        error_msg_html = format_html(
+                            _("It was not possible to import the following items:")
+                            + f"<div>{error_msg_html}</div>"
+                        )
+                        self.message_user(request, error_msg_html, messages.ERROR)
+
+                    return HttpResponseRedirect(
+                        reverse(
+                            "admin:openzaak_zaaktypeconfig_changelist",
+                        )
+                    )
+                except Exception:
+                    logger.exception("Unable to process ZGW import")
+                    self.message_user(
+                        request,
+                        _(
+                            "We were unable to process your upload. Please regenerate the file and try again."
+                        ),
+                        messages.ERROR,
+                    )
+                finally:
+                    storage.delete(target_file_name)
+
+        return TemplateResponse(
+            request, "admin/import_zgw_export_form.html", {"form": form}
+        )
+
     def has_add_permission(self, request):
         return False
 

src/open_inwoner/openzaak/import_export.py

@@ -166,19 +166,17 @@ def _update_nested_zgw_config(
 
 
 @dataclasses.dataclass(frozen=True)
-class CatalogusConfigExport:
-    """Gather and export CatalogusConfig(s) and all associated relations."""
-
+class ZGWConfigExport:
     catalogus_configs: QuerySet
-    zaak_type_configs: QuerySet
+    zaaktype_configs: QuerySet
     zaak_informatie_object_type_configs: QuerySet
     zaak_status_type_configs: QuerySet
     zaak_resultaat_type_configs: QuerySet
 
     def __iter__(self) -> Generator[QuerySet, Any, None]:
         yield from (
             self.catalogus_configs,
-            self.zaak_type_configs,
+            self.zaaktype_configs,
             self.zaak_informatie_object_type_configs,
             self.zaak_status_type_configs,
             self.zaak_resultaat_type_configs,
@@ -188,9 +186,32 @@ def __eq__(self, other: QuerySet) -> bool:
         for a, b in zip(self, other):
             if a.difference(b).exists():
                 return False
-
         return True
 
+    def as_dicts_iter(self) -> Generator[dict, Any, None]:
+        for qs in self:
+            serialized_data = serializers.serialize(
+                queryset=qs,
+                format="json",
+                use_natural_foreign_keys=True,
+                use_natural_primary_keys=True,
+            )
+            json_data: list[dict] = json.loads(
+                serialized_data,
+            )
+            yield from json_data
+
+    def as_jsonl_iter(self) -> Generator[str, Any, None]:
+        for row in self.as_dicts():
+            yield json.dumps(row)
+            yield "\n"
+
+    def as_dicts(self) -> list[dict]:
+        return list(self.as_dicts_iter())
+
+    def as_jsonl(self) -> str:
+        return "".join(self.as_jsonl_iter())
+
     @classmethod
     def from_catalogus_configs(cls, catalogus_configs: QuerySet) -> Self:
         if not isinstance(catalogus_configs, QuerySet):
@@ -203,54 +224,60 @@ def from_catalogus_configs(cls, catalogus_configs: QuerySet) -> Self:
                 f"`catalogus_configs` is of type {catalogus_configs.model}, not CatalogusConfig"
             )
 
-        zaak_type_configs = ZaakTypeConfig.objects.filter(
+        zaaktype_configs = ZaakTypeConfig.objects.filter(
             catalogus__in=catalogus_configs
         )
         informatie_object_types = ZaakTypeInformatieObjectTypeConfig.objects.filter(
-            zaaktype_config__in=zaak_type_configs
+            zaaktype_config__in=zaaktype_configs
         )
         zaak_status_type_configs = ZaakTypeStatusTypeConfig.objects.filter(
-            zaaktype_config__in=zaak_type_configs
+            zaaktype_config__in=zaaktype_configs
         )
         zaak_resultaat_type_configs = ZaakTypeResultaatTypeConfig.objects.filter(
-            zaaktype_config__in=zaak_type_configs
+            zaaktype_config__in=zaaktype_configs
         )
 
         return cls(
             catalogus_configs=catalogus_configs,
-            zaak_type_configs=zaak_type_configs,
+            zaaktype_configs=zaaktype_configs,
             zaak_informatie_object_type_configs=informatie_object_types,
             zaak_status_type_configs=zaak_status_type_configs,
             zaak_resultaat_type_configs=zaak_resultaat_type_configs,
         )
 
-    def as_dicts_iter(self) -> Generator[dict, Any, None]:
-        for qs in self:
-            serialized_data = serializers.serialize(
-                queryset=qs,
-                format="json",
-                use_natural_foreign_keys=True,
-                use_natural_primary_keys=True,
-            )
-            json_data: list[dict] = json.loads(
-                serialized_data,
+    @classmethod
+    def from_zaaktype_configs(cls, zaaktype_configs: QuerySet) -> Self:
+        if not isinstance(zaaktype_configs, QuerySet):
+            raise TypeError(
+                f"`zaaktype_configs` is not a QuerySet, but a {type(zaaktype_configs)}"
             )
-            yield from json_data
 
-    def as_jsonl_iter(self) -> Generator[str, Any, None]:
-        for row in self.as_dicts():
-            yield json.dumps(row)
-            yield "\n"
+        if zaaktype_configs.model != ZaakTypeConfig:
+            raise ValueError(
+                f"`zaaktype_configs` is of type {zaaktype_configs.model}, not ZaakTypeConfig"
+            )
 
-    def as_dicts(self) -> list[dict]:
-        return list(self.as_dicts_iter())
+        informatie_object_types = ZaakTypeInformatieObjectTypeConfig.objects.filter(
+            zaaktype_config__in=zaaktype_configs
+        )
+        zaak_status_type_configs = ZaakTypeStatusTypeConfig.objects.filter(
+            zaaktype_config__in=zaaktype_configs
+        )
+        zaak_resultaat_type_configs = ZaakTypeResultaatTypeConfig.objects.filter(
+            zaaktype_config__in=zaaktype_configs
+        )
 
-    def as_jsonl(self) -> str:
-        return "".join(self.as_jsonl_iter())
+        return cls(
+            catalogus_configs=CatalogusConfig.objects.none(),
+            zaaktype_configs=zaaktype_configs,
+            zaak_informatie_object_type_configs=informatie_object_types,
+            zaak_status_type_configs=zaak_status_type_configs,
+            zaak_resultaat_type_configs=zaak_resultaat_type_configs,
+        )
 
 
 @dataclasses.dataclass(frozen=True)
-class CatalogusConfigImport:
+class ZGWConfigImport:
     """Import CatalogusConfig(s) and all associated relations."""
 
     total_rows_processed: int = 0

src/open_inwoner/openzaak/tests/test_admin.py

@@ -192,7 +192,7 @@ def test_import_flow_reports_success(self) -> None:
 
         form = self.app.get(
             reverse(
-                "admin:upload_zgw_import_file",
+                "admin:upload_catalogus_import_file",
             ),
             user=self.user,
         ).form
@@ -219,13 +219,13 @@ def test_import_flow_reports_success(self) -> None:
         )
 
     @mock.patch(
-        "open_inwoner.openzaak.import_export.CatalogusConfigImport.import_from_jsonl_file_in_django_storage"
+        "open_inwoner.openzaak.import_export.ZGWConfigImport.import_from_jsonl_file_in_django_storage"
     )
     def test_import_flow_errors_reports_failure_to_user(self, m) -> None:
         m.side_effect = Exception("something went wrong")
         form = self.app.get(
             reverse(
-                "admin:upload_zgw_import_file",
+                "admin:upload_catalogus_import_file",
             ),
             user=self.user,
         ).form
@@ -255,7 +255,7 @@ def test_import_flow_errors_reports_failure_to_user(self, m) -> None:
         self.assertEqual(
             response.request.path,
             reverse(
-                "admin:upload_zgw_import_file",
+                "admin:upload_catalogus_import_file",
             ),
         )
 
@@ -273,7 +273,7 @@ def test_import_flow_reports_errors(self) -> None:
 
         form = self.app.get(
             reverse(
-                "admin:upload_zgw_import_file",
+                "admin:upload_catalogus_import_file",
             ),
             user=self.user,
         ).form
@@ -334,7 +334,7 @@ def test_import_flow_reports_partial_errors(self) -> None:
 
         form = self.app.get(
             reverse(
-                "admin:upload_zgw_import_file",
+                "admin:upload_catalogus_import_file",
             ),
             user=self.user,
         ).form