diff --git a/Pipfile b/Pipfile index 5dd9e71b65..590a56effd 100644 --- a/Pipfile +++ b/Pipfile @@ -45,7 +45,6 @@ requests = "==2.31.0" sentry-sdk = "==1.30.0" whitenoise = "==6.5.0" redis-om = "==0.2.1" -django-ulid = "==0.0.4" [dev-packages] black = "==23.9.1" diff --git a/Pipfile.lock b/Pipfile.lock index 9d0a6e2a66..6cdb09ff6a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1e1157603bf6e3109439f534191f4cadb995d51a679755c206baf24649e6266f" + "sha256": "e46c7dd4bb313bf57fc2b800581aab91c31c2fdf7599dafebac09b66bb22208c" }, "pipfile-spec": 6, "requires": { @@ -468,15 +468,6 @@ "markers": "python_version >= '3.7'", "version": "==3.3.0" }, - "django-ulid": { - "hashes": [ - "sha256:56c3b302c740d01c29677f65390c72648afbe3a3109e1077ddaac2add254cbb3", - "sha256:886be1fa873d28617a51cb0c1b4d9946fff30889ef031e546ac407d8160dbb74" - ], - "index": "pypi", - "markers": "python_version >= '3.5'", - "version": "==0.0.4" - }, "djangoql": { "hashes": [ "sha256:4f053d0128e28f412926e2da902735f4bdcbab5c08d43be4dfefd747fca2e96e" @@ -1306,13 +1297,6 @@ "markers": "python_version >= '2'", "version": "==2024.1" }, - "ulid-py": { - "hashes": [ - "sha256:b56a0f809ef90d6020b21b89a87a48edc7c03aea80e5ed5174172e82d76e3987", - "sha256:dc6884be91558df077c3011b9fb0c87d1097cb8fc6534b11f310161afd5738f0" - ], - "version": "==1.1.0" - }, "unicodecsv": { "hashes": [ "sha256:018c08037d48649a0412063ff4eda26eaa81eff1546dbffa51fa5293276ff7fc" @@ -1332,7 +1316,7 @@ "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e" ], - "markers": "python_version >= '3.7'", + "markers": "python_version >= '3.6'", "version": "==2.0.7" }, "vine": { @@ -1437,6 +1421,14 @@ } }, "develop": { + "appnope": { + "hashes": [ + "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", + "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c" + ], + "markers": "sys_platform == 'darwin'", + "version": "==0.1.4" + }, "asgiref": { "hashes": [ "sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e", @@ -2358,7 +2350,7 @@ "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e" ], - "markers": "python_version >= '3.7'", + "markers": "python_version >= '3.6'", "version": "==2.0.7" }, "virtualenv": { @@ -2766,7 +2758,7 @@ "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e" ], - "markers": "python_version >= '3.7'", + "markers": "python_version >= '3.6'", "version": "==2.0.7" } } diff --git a/care/facility/api/serializers/events.py b/care/facility/api/serializers/events.py index 70dfce3fc6..5378d956ba 100644 --- a/care/facility/api/serializers/events.py +++ b/care/facility/api/serializers/events.py @@ -1,7 +1,7 @@ -from django_ulid.serializers import ULIDField from rest_framework.serializers import ModelSerializer, SerializerMethodField from care.facility.models.events import EventType, PatientConsultationEvent +from care.utils.ulid.serializers import ULIDField class EventTypeSerializer(ModelSerializer): diff --git a/care/facility/migrations/0408_patientnotesedit.py b/care/facility/migrations/0408_patientnotesedit.py index b3743adb55..aa6f35d22a 100644 --- a/care/facility/migrations/0408_patientnotesedit.py +++ b/care/facility/migrations/0408_patientnotesedit.py @@ -10,7 +10,7 @@ def create_initial_patient_notes_edit_record(apps, schema_editor): PatientNotes = apps.get_model("facility", "PatientNotes") PatientNotesEdit = apps.get_model("facility", "PatientNotesEdit") - notes_without_edits = PatientNotes.objects.all() + notes_without_edits = PatientNotes.objects.all().order_by("pk") paginator = Paginator(notes_without_edits, 1000) for page_number in paginator.page_range: diff --git a/care/facility/migrations/0410_availabilityrecord_delete_assetavailabilityrecord_and_more.py b/care/facility/migrations/0410_availabilityrecord_delete_assetavailabilityrecord_and_more.py index 77b024d683..87e989c5c4 100644 --- a/care/facility/migrations/0410_availabilityrecord_delete_assetavailabilityrecord_and_more.py +++ b/care/facility/migrations/0410_availabilityrecord_delete_assetavailabilityrecord_and_more.py @@ -17,7 +17,7 @@ def forwards_func(apps, schema_editor): asset_content_type = ContentType.objects.get_for_model(Asset) - aar_records = AssetAvailabilityRecord.objects.all() + aar_records = AssetAvailabilityRecord.objects.all().order_by("pk") paginator = Paginator(aar_records, 1000) for page_number in paginator.page_range: @@ -41,7 +41,9 @@ def backwards_func(apps, schema_editor): asset_content_type = ContentType.objects.get_for_model(Asset) - ar_records = AvailabilityRecord.objects.filter(content_type=asset_content_type) + ar_records = AvailabilityRecord.objects.filter( + content_type=asset_content_type + ).order_by("pk") paginator = Paginator(ar_records, 1000) for page_number in paginator.page_range: @@ -121,7 +123,6 @@ class Migration(migrations.Migration): ], options={ "ordering": ["-timestamp"], - "unique_together": {("object_external_id", "timestamp")}, }, ), migrations.AddIndex( @@ -131,10 +132,6 @@ class Migration(migrations.Migration): name="facility_av_content_ad9eff_idx", ), ), - migrations.AlterUniqueTogether( - name="availabilityrecord", - unique_together={("object_external_id", "timestamp")}, - ), migrations.RunPython(forwards_func, backwards_func), migrations.DeleteModel( name="AssetAvailabilityRecord", diff --git a/care/facility/migrations/0411_merge_20240212_1429.py b/care/facility/migrations/0411_merge_20240212_1429.py new file mode 100644 index 0000000000..344959e5bb --- /dev/null +++ b/care/facility/migrations/0411_merge_20240212_1429.py @@ -0,0 +1,12 @@ +# Generated by Django 4.2.10 on 2024-02-12 08:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("facility", "0408_patientnotesedit"), + ("facility", "0410_availabilityrecord_delete_assetavailabilityrecord_and_more"), + ] + + operations = [] diff --git a/care/facility/migrations/0412_availabilityrecord_unique_constraint.py b/care/facility/migrations/0412_availabilityrecord_unique_constraint.py new file mode 100644 index 0000000000..0be6345fcd --- /dev/null +++ b/care/facility/migrations/0412_availabilityrecord_unique_constraint.py @@ -0,0 +1,42 @@ +from django.db import migrations, models + + +def clean_duplicates(apps, schema_editor): + AvailabilityRecord = apps.get_model("facility", "AvailabilityRecord") + + duplicates = ( + AvailabilityRecord.objects.values("object_external_id", "timestamp") + .annotate(count=models.Count("id")) + .filter(count__gt=1) + ) + + ids_to_delete = [] + for duplicate in duplicates: + record_ids = list( + AvailabilityRecord.objects.filter( + object_external_id=duplicate["object_external_id"], + timestamp=duplicate["timestamp"], + ).values_list("id", flat=True) + ) + + ids_to_delete.extend(record_ids[1:]) + + if ids_to_delete: + AvailabilityRecord.objects.filter(id__in=set(ids_to_delete)).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("facility", "0411_merge_20240212_1429"), + ] + + operations = [ + migrations.RunPython(clean_duplicates, reverse_code=migrations.RunPython.noop), + migrations.AddConstraint( + model_name="availabilityrecord", + constraint=models.UniqueConstraint( + fields=("object_external_id", "timestamp"), + name="object_external_id_timestamp_unique", + ), + ), + ] diff --git a/care/facility/migrations/0410_eventtype_patientconsultationevent.py b/care/facility/migrations/0413_eventtype_patientconsultationevent_and_more.py similarity index 59% rename from care/facility/migrations/0410_eventtype_patientconsultationevent.py rename to care/facility/migrations/0413_eventtype_patientconsultationevent_and_more.py index fb4393b27b..7fdd9bbca1 100644 --- a/care/facility/migrations/0410_eventtype_patientconsultationevent.py +++ b/care/facility/migrations/0413_eventtype_patientconsultationevent_and_more.py @@ -1,19 +1,19 @@ -# Generated by Django 4.2.8 on 2024-02-10 14:40 +# Generated by Django 4.2.10 on 2024-02-12 10:31 import django.contrib.postgres.fields import django.db.models.deletion -import django_ulid.models -import ulid.api.api from django.conf import settings from django.db import migrations, models import care.utils.event_utils +import care.utils.ulid.models +import care.utils.ulid.ulid class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ("facility", "0409_merge_20240210_1510"), + ("facility", "0412_availabilityrecord_unique_constraint"), ] operations = [ @@ -42,15 +42,6 @@ class Migration(migrations.Migration): ), ("created_date", models.DateTimeField(auto_now_add=True)), ("is_active", models.BooleanField(default=True)), - ( - "parent", - models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.PROTECT, - related_name="children", - to="facility.eventtype", - ), - ), ], ), migrations.CreateModel( @@ -67,8 +58,8 @@ class Migration(migrations.Migration): ), ( "external_id", - django_ulid.models.ULIDField( - default=ulid.api.api.Api.new, editable=False, unique=True + care.utils.ulid.models.ULIDField( + default=care.utils.ulid.ulid.ULID, editable=False, unique=True ), ), ("created_date", models.DateTimeField(db_index=True)), @@ -99,37 +90,60 @@ class Migration(migrations.Migration): max_length=10, ), ), - ( - "caused_by", - models.ForeignKey( - on_delete=django.db.models.deletion.PROTECT, - to=settings.AUTH_USER_MODEL, - ), - ), - ( - "consultation", - models.ForeignKey( - on_delete=django.db.models.deletion.PROTECT, - related_name="events", - to="facility.patientconsultation", - ), - ), - ( - "event_type", - models.ForeignKey( - on_delete=django.db.models.deletion.PROTECT, - to="facility.eventtype", - ), - ), ], options={ "ordering": ["-created_date"], - "indexes": [ - models.Index( - fields=["consultation", "is_latest"], - name="facility_pa_consult_7b22fe_idx", - ) - ], }, ), + migrations.RemoveConstraint( + model_name="availabilityrecord", + name="object_external_id_timestamp_unique", + ), + migrations.AddConstraint( + model_name="availabilityrecord", + constraint=models.UniqueConstraint( + fields=("object_external_id", "timestamp"), + name="object_external_id_timestamp", + ), + ), + migrations.AddField( + model_name="patientconsultationevent", + name="caused_by", + field=models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL + ), + ), + migrations.AddField( + model_name="patientconsultationevent", + name="consultation", + field=models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="events", + to="facility.patientconsultation", + ), + ), + migrations.AddField( + model_name="patientconsultationevent", + name="event_type", + field=models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to="facility.eventtype" + ), + ), + migrations.AddField( + model_name="eventtype", + name="parent", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="children", + to="facility.eventtype", + ), + ), + migrations.AddIndex( + model_name="patientconsultationevent", + index=models.Index( + fields=["consultation", "is_latest"], + name="facility_pa_consult_7b22fe_idx", + ), + ), ] diff --git a/care/facility/models/asset.py b/care/facility/models/asset.py index 0804442da7..af560e8e9c 100644 --- a/care/facility/models/asset.py +++ b/care/facility/models/asset.py @@ -209,7 +209,12 @@ class Meta: indexes = [ models.Index(fields=["content_type", "object_external_id"]), ] - unique_together = (("object_external_id", "timestamp"),) + constraints = [ + models.UniqueConstraint( + name="object_external_id_timestamp", + fields=["object_external_id", "timestamp"], + ) + ] ordering = ["-timestamp"] def __str__(self): diff --git a/care/facility/models/events.py b/care/facility/models/events.py index 844c567fbb..e502677ff5 100644 --- a/care/facility/models/events.py +++ b/care/facility/models/events.py @@ -1,9 +1,10 @@ from django.contrib.auth import get_user_model from django.contrib.postgres.fields import ArrayField from django.db import models -from django_ulid.models import ULIDField, default from care.utils.event_utils import CustomJSONEncoder +from care.utils.ulid.models import ULIDField +from care.utils.ulid.ulid import ULID User = get_user_model() @@ -46,7 +47,7 @@ def __str__(self) -> str: class PatientConsultationEvent(models.Model): - external_id = ULIDField(default=default, editable=False, unique=True) + external_id = ULIDField(default=ULID, editable=False, unique=True) consultation = models.ForeignKey( "PatientConsultation", on_delete=models.PROTECT, diff --git a/care/utils/ulid/__init__.py b/care/utils/ulid/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/care/utils/ulid/models.py b/care/utils/ulid/models.py new file mode 100644 index 0000000000..9305735e30 --- /dev/null +++ b/care/utils/ulid/models.py @@ -0,0 +1,43 @@ +from django.core import exceptions +from django.db import models + +from .ulid import ULID + + +class ULIDField(models.Field): + description = "Universally Unique Lexicographically Sortable Identifier" + empty_strings_allowed = False + + def __init__(self, verbose_name=None, **kwargs): + kwargs.setdefault("max_length", 26) # default length of ulid + super().__init__(verbose_name, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + del kwargs["max_length"] + return name, path, args, kwargs + + def get_internal_type(self) -> str: + return "CharField" + + def get_db_prep_value(self, value, connection, prepared=False) -> str | None: + if value is None: + return None + if not isinstance(value, ULID): + value = self.to_python(value) + return str(value) + + def from_db_value(self, value, expression, connection) -> ULID | None: + return self.to_python(value) + + def to_python(self, value) -> ULID | None: + if value is None: + return None + try: + return ULID.parse(value) + except (AttributeError, ValueError): + raise exceptions.ValidationError( + self.error_messages["invalid"], + code="invalid", + params={"value": value}, + ) diff --git a/care/utils/ulid/serializers.py b/care/utils/ulid/serializers.py new file mode 100644 index 0000000000..2b89cf645f --- /dev/null +++ b/care/utils/ulid/serializers.py @@ -0,0 +1,19 @@ +from django.utils.translation import gettext as _ +from rest_framework import fields + +from .ulid import ULID + + +class ULIDField(fields.Field): + default_error_messages = { + "invalid": _('"{value}" is not a valid ULID.'), + } + + def to_internal_value(self, data) -> ULID: + try: + return ULID.parse(data) + except (AttributeError, ValueError): + self.fail("invalid", value=data) + + def to_representation(self, value) -> str: + return str(ULID.parse(value)) diff --git a/care/utils/ulid/ulid.py b/care/utils/ulid/ulid.py new file mode 100644 index 0000000000..0bee58bf11 --- /dev/null +++ b/care/utils/ulid/ulid.py @@ -0,0 +1,35 @@ +from typing import Self +from uuid import UUID + +from ulid import ULID as BaseULID + + +class ULID(BaseULID): + @classmethod + def parse(cls, value) -> Self: + if isinstance(value, BaseULID): + return value + if isinstance(value, UUID): + return cls.from_uuid(value) + if isinstance(value, str): + len_value = len(value) + if len_value == 32 or len_value == 36: + return cls.from_uuid(UUID(value)) + if len_value == 26: + return cls.from_str(value) + if len_value == 16: + return cls.from_bytes(value) + if len_value == 10: + return cls.from_timestamp(int(value)) + raise ValueError( + "Cannot create ULID from string of length {}".format(len_value) + ) + if isinstance(value, (int, float)): + return cls.from_int(int(value)) + if isinstance(value, (bytes, bytearray)): + return cls.from_bytes(value) + if isinstance(value, memoryview): + return cls.from_bytes(value.tobytes()) + raise ValueError( + "Cannot create ULID from type {}".format(value.__class__.__name__) + )