Support uploading model runs from the web client (#490)

* Support uploading model runs from the web client

* Fix poetry lockfile

* Fix migration script

* Handle VFileInput model as File | File[]

* Skip originator validation

* Use s3 in prod, minio in dev

* Refresh performers and regions on upload

* Apply overrides to models directly

Also adds more restricted conditions for overriding

* Disable all inputs when upload is loading

* Merge updateRegionMap into updateRegionList

* Fix UI reset and incorrect file validation logic

* Use delay_on_commit

* Revert "skip validation for SiteFeature"

* Return the task ID and refresh the model run list

* Remove unused import

* Run lint:fix npm script

* Recommended UX tweaks

* Fix migration dependency

* Use single update query to avoid race cond

Co-authored-by: Mike VanDenburgh <37340715+mvandenburgh@users.noreply.github.com>

* Delete the upload object

---------

Co-authored-by: Mike VanDenburgh <37340715+mvandenburgh@users.noreply.github.com>

pyproject.toml

@@ -60,6 +60,7 @@ apache-airflow-client = "^2.9.0"
 beautifulsoup4 = "^4.12.3"
 django-allauth = {extras = ["socialaccount"], version = "^0.63.2"}
 django-login-required-middleware = "^0.9.0"
+django-s3-file-field = {version = "^1.0.1", extras = ["s3"]}
 
 [tool.poetry.group.dev.dependencies]
 django-stubs = "^4.2.7"
@@ -68,6 +69,7 @@ tox = "^4.14.2"
 django-minio-storage = "^0.5.7"
 werkzeug = "^3.0.2"
 django-debug-toolbar = "^4.3.0"
+django-s3-file-field = {version = "^1.0.1", extras = ["minio"]}
 
 [tool.poetry.group.dev]
 optional = true

rdwatch/core/admin.py

@@ -4,6 +4,7 @@
     AnimationModelRunExport,
     AnimationSiteExport,
     ModelRun,
+    ModelRunUpload,
     Performer,
     Region,
     SatelliteFetching,
@@ -151,3 +152,15 @@ class AnimationModelRunExportAdmin(admin.ModelAdmin):
         'celery_id',
         'arguments',
     )
+
+
+@admin.register(ModelRunUpload)
+class ModelRunUploadAdmin(admin.ModelAdmin):
+    list_display = (
+        'id',
+        'title',
+        'performer',
+        'region',
+        'zipfile',
+        'task_id',
+    )

rdwatch/core/migrations/0036_modelrunupload.py

@@ -0,0 +1,59 @@
+# Generated by Django 5.0.8 on 2024-08-14 21:17
+
+import uuid
+
+import s3_file_field.fields
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('core', '0035_alter_annotationexport_export_file_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ModelRunUpload',
+            fields=[
+                (
+                    'id',
+                    models.UUIDField(
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                ('title', models.CharField(max_length=1000)),
+                (
+                    'private',
+                    models.BooleanField(
+                        default=False,
+                        help_text='Whether this model run should be private',
+                    ),
+                ),
+                (
+                    'region',
+                    models.CharField(
+                        blank=True,
+                        help_text='Override for the region this model run belongs to',
+                        max_length=1000,
+                    ),
+                ),
+                (
+                    'performer',
+                    models.CharField(
+                        blank=True,
+                        help_text='Shortcode override for the team that produced this evaluation',
+                        max_length=1000,
+                    ),
+                ),
+                ('zipfile', s3_file_field.fields.S3FileField()),
+                (
+                    'task_id',
+                    models.CharField(help_text='Celery task ID', max_length=256),
+                ),
+            ],
+        ),
+    ]

rdwatch/core/models/__init__.py

@@ -5,6 +5,7 @@
     AnnotationExport,
 )
 from .model_run import ModelRun
+from .model_run_upload import ModelRunUpload
 from .performer import Performer
 from .region import Region
 from .satellite_fetching import SatelliteFetching
@@ -16,6 +17,7 @@
     'AnnotationExport',
     'lookups',
     'ModelRun',
+    'ModelRunUpload',
     'Performer',
     'Region',
     'SiteEvaluation',

rdwatch/core/models/model_run_upload.py

@@ -0,0 +1,38 @@
+from uuid import uuid4
+
+from s3_file_field import S3FileField
+
+from django.db import models
+from django.db.models.signals import pre_delete
+from django.dispatch import receiver
+
+
+class ModelRunUpload(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
+
+    title = models.CharField(max_length=1000)
+    private = models.BooleanField(
+        default=False, help_text='Whether this model run should be private'
+    )
+    region = models.CharField(
+        max_length=1000,
+        blank=True,
+        help_text='Override for the region this model run belongs to',
+    )
+    performer = models.CharField(
+        max_length=1000,
+        blank=True,
+        help_text='Shortcode override for the team that produced this evaluation',
+    )
+    zipfile = S3FileField()
+
+    task_id = models.CharField(max_length=256, help_text='Celery task ID')
+
+    def __str__(self) -> str:
+        return f'<ModelRunUpload {self.id}>'
+
+
+@receiver(pre_delete, sender=ModelRunUpload)
+def delete_zipfile(sender, instance, **kwargs):
+    if instance.zipfile:
+        instance.zipfile.delete(save=False)