[#310] Improve migrations

maykinmedia · Jan 28, 2025 · 4a340c9 · 4a340c9
1 parent 0aa2b2d
commit 4a340c9
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 87 deletions.
diff --git a/...s/contactgegevens/migrations/0005_alter_organisatie_adres_nummeraanduiding_id_and_more.py b/...s/contactgegevens/migrations/0005_alter_organisatie_adres_nummeraanduiding_id_and_more.py
@@ -1,58 +1,44 @@
-# Generated by Django 4.2.15 on 2025-01-09 09:22
+# Generated by Django 4.2.17 on 2025-01-28 09:10
 
-import json
 import logging
 import openklant.utils.validators
+from django.core.exceptions import ValidationError
 from django.core.management import CommandError
 from django.db import migrations, models
-from django.db.models.functions import Length
+from openklant.utils.validators import validate_bag_id
 
 logger = logging.getLogger(__name__)
 
 
-def _log_records(records, model_name):
-    logger.info("--- START %s INFO ---", model_name)
-    logger.info(
-        "Found %s records for %s that have adres_nummeraanduiding_id length > 16 chars",
-        records.count(),
-        model_name,
-    )
-
+def _check_records(records, model_name):
+    total_failed_records = 0
     for record in records:
-        logger.warning(
-            json.dumps(
-                {
-                    "model_name": model_name,
-                    "pk": record.pk,
-                    "uuid": str(record.uuid),
-                    "adres_nummeraanduiding_id": record.adres_nummeraanduiding_id,
-                }
+        try:
+            validate_bag_id(record.adres_nummeraanduiding_id)
+        except ValidationError:
+            logger.warning(
+                "%s(pk=%s, uuid=%s) Field: 'adres_nummeraanduiding_id'. Invalid BAG ID: %s",
+                model_name,
+                record.pk,
+                record.uuid,
+                record.adres_nummeraanduiding_id,
             )
-        )
-
-    logger.info("--- END %s INFO ---", model_name)
+            total_failed_records += 1
+    return total_failed_records
 
 
-def _check_field_length(apps, schema_editor):
-
+def _check_records_field_length(apps, schema_editor):
     Organisatie = apps.get_model("contactgegevens", "Organisatie")
     Persoon = apps.get_model("contactgegevens", "Persoon")
 
-    organisatie_queryset = Organisatie.objects.annotate(
-        length=Length("adres_nummeraanduiding_id")
-    ).filter(length__gt=16)
-
-    persoon_queryset = Persoon.objects.annotate(
-        length=Length("adres_nummeraanduiding_id")
-    ).filter(length__gt=16)
-
-    _log_records(organisatie_queryset, "Organisatie")
-    _log_records(persoon_queryset, "Persoon")
-
-    if organisatie_queryset or persoon_queryset:
-        raise CommandError(
-            "The migration cannot proceed due to data that doesn't comply with the model's requirements."
-        )
+    for model_name, model in [("Organisatie", Organisatie), ("Persoon", Persoon)]:
+        records = model.objects.all()
+        if total_failed_records := _check_records(records, model_name):
+            raise CommandError(
+                "The migration cannot proceed due to %s records that don't comply with the %s model's requirements. "
+                "Possible data inconsistency or mapping error."
+                % (total_failed_records, model_name)
+            )
 
 
 class Migration(migrations.Migration):
@@ -66,7 +52,7 @@ class Migration(migrations.Migration):
 
     operations = [
         migrations.RunPython(
-            code=_check_field_length,
+            code=_check_records_field_length,
             reverse_code=migrations.RunPython.noop,
         ),
         migrations.AlterField(

diff --git a/...tinteracties/migrations/0027_alter_betrokkene_bezoekadres_nummeraanduiding_id_and_more.py b/...tinteracties/migrations/0027_alter_betrokkene_bezoekadres_nummeraanduiding_id_and_more.py
@@ -1,62 +1,54 @@
-# Generated by Django 4.2.17 on 2025-01-27 15:06
+# Generated by Django 4.2.17 on 2025-01-28 09:10
 
-import json
 import logging
 import openklant.utils.validators
+from django.core.exceptions import ValidationError
 from django.core.management import CommandError
 from django.db import migrations, models
-from django.db.models.functions import Length
+from openklant.utils.validators import validate_bag_id
 
 logger = logging.getLogger(__name__)
 
 
-def _log_records(records, model_name):
-    logger.info("--- START %s INFO ---", model_name)
-    logger.info(
-        "Found %s records for %s that have bezoekadres_nummeraanduiding_id "
-        "or correspondentieadres_nummeraanduiding_id length > 16 chars",
-        records.count(),
-        model_name,
-    )
-
+def _check_records(records, model_name):
+    total_failed_records = 0
     for record in records:
-        logger.warning(
-            json.dumps(
-                {
-                    "model_name": model_name,
-                    "pk": record.pk,
-                    "uuid": str(record.uuid),
-                    "bezoekadres_nummeraanduiding_id": record.bezoekadres_nummeraanduiding_id,
-                    "correspondentieadres_nummeraanduiding_id": record.correspondentieadres_nummeraanduiding_id,
-                }
-            )
-        )
-
-    logger.info("--- END %s INFO ---", model_name)
-
-
-def _check_field_length(apps, schema_editor):
-
+        record_failed = False
+        for field_name in (
+            "bezoekadres_nummeraanduiding_id",
+            "correspondentieadres_nummeraanduiding_id",
+        ):
+            try:
+                field_value = getattr(record, field_name, None)
+                validate_bag_id(field_value)
+            except ValidationError:
+                logger.warning(
+                    "%s(pk=%s, uuid=%s) Field: '%s'. Invalid BAG ID: %s",
+                    model_name,
+                    record.pk,
+                    record.uuid,
+                    field_name,
+                    field_value,
+                )
+                record_failed = True
+
+        if record_failed:
+            total_failed_records += 1
+    return total_failed_records
+
+
+def _check_records_field_length(apps, schema_editor):
     Betrokkene = apps.get_model("klantinteracties", "Betrokkene")
     Partij = apps.get_model("klantinteracties", "Partij")
 
-    betrokkene_queryset = Betrokkene.objects.annotate(
-        length1=Length("bezoekadres_nummeraanduiding_id"),
-        length2=Length("correspondentieadres_nummeraanduiding_id"),
-    ).filter(models.Q(length1__gt=16) | models.Q(length2__gt=16))
-
-    partij_queryset = Partij.objects.annotate(
-        length1=Length("bezoekadres_nummeraanduiding_id"),
-        length2=Length("correspondentieadres_nummeraanduiding_id"),
-    ).filter(models.Q(length1__gt=16) | models.Q(length2__gt=16))
-
-    _log_records(betrokkene_queryset, "Betrokkene")
-    _log_records(partij_queryset, "Partij")
-
-    if betrokkene_queryset or partij_queryset:
-        raise CommandError(
-            "The migration cannot proceed due to data that doesn't comply with the model's requirements."
-        )
+    for model_name, model in [("Betrokkene", Betrokkene), ("Partij", Partij)]:
+        records = model.objects.all()
+        if total_failed_records := _check_records(records, model_name):
+            raise CommandError(
+                "The migration cannot proceed due to %s records that don't comply with the %s model's requirements. "
+                "Possible data inconsistency or mapping error."
+                % (total_failed_records, model_name)
+            )
 
 
 class Migration(migrations.Migration):
@@ -67,7 +59,7 @@ class Migration(migrations.Migration):
 
     operations = [
         migrations.RunPython(
-            code=_check_field_length,
+            code=_check_records_field_length,
             reverse_code=migrations.RunPython.noop,
         ),
         migrations.AlterField(