initial work on file upload - DO NOT MERGE

I'm backing this up online as a draft PR. This has the functionality
to let a user upload files from an incident into the filesystem on
the IMS server. There's more to do.

#7

pyproject.toml

@@ -21,6 +21,8 @@ dependencies = [
     "hyperlink==21.0.0",
     "jwcrypto==1.5.6",
     "klein==24.8.0",
+    "multipart>=1.2.1",
+    "puremagic>=1.28",
     "pymysql==1.1.1",
     "pyopenssl==25.0.0",
     "pyyaml==6.0.2",
@@ -106,6 +108,7 @@ ignore = [
     "EM102",    # Exception must not use an f-string literal, assign to variable first
     "ERA001",   # Found commented-out code
     "FIX001",   # Line contains FIXME, consider resolving the issue
+    "FIX002",   # Line contains TODO, consider resolving the issue
     "ISC001",   # Implicitly concatenated strings on a single line  # Disabled for formatter compatibility
     "N802",     # Function name should be lowercase
     "N803",     # Argument name should be lowercase

src/ims/application/_api.py

@@ -30,12 +30,17 @@
 from enum import Enum
 from functools import partial
 from json import JSONDecodeError
+from pathlib import Path
 from typing import Any, ClassVar, cast
+from uuid import uuid4
 
+import multipart
+import puremagic
 from attrs import frozen
 from hyperlink import URL
 from klein import KleinRenderable
 from klein._app import KleinSynchronousRenderable
+from puremagic import PureError
 from twisted.internet.defer import Deferred
 from twisted.internet.error import ConnectionDone
 from twisted.logger import Logger
@@ -684,6 +689,80 @@ def _cast(obj: Any) -> Any:
 
         return noContentResponse(request)
 
+    @router.route(_unprefix(URLs.incidentAttach), methods=("POST",))
+    async def attachFileToIncident(
+        self,
+        request: IRequest,
+        event_id: str,
+        incident_number: str,
+    ) -> KleinSynchronousRenderable:
+        eventId = event_id
+        incidentNumber = int(incident_number)
+        del event_id
+        del incident_number
+
+        await self.config.authProvider.authorizeRequest(
+            request, eventId, Authorization.writeIncidents
+        )
+
+        _, options = multipart.parse_options_header(request.getHeader("Content-Type"))
+        p = multipart.MultipartParser(
+            request.content,
+            options.get("boundary", ""),
+            # TODO: move this to config
+            # Arbitrary limit on how many files per attach API call
+            part_limit=50,
+            # TODO: move this to config
+            # Arbitrary 10 MB per file limit
+            partsize_limit=10 * 1024 * 1024,
+        )
+
+        store = self.config.store
+
+        eventNum = 0
+        for event in await store.events():
+            if event.id == eventId:
+                eventNum = event.number
+
+        # TODO: move this to config
+        eventsDir = Path.home() / "ims-attachments" / "events"
+        for part in p.parts():
+            newFilename = uuid4()
+            try:
+                extension = puremagic.from_string(part.raw, filename=part.filename)
+            except PureError:
+                self._log.info(f"failed to determine filetype for {part.filename}")
+                extension = ""
+            else:
+                self._log.info(
+                    f"detected file extension {extension} for {part.filename}"
+                )
+
+            incidentDir = eventsDir / str(eventNum) / "incidents" / str(incidentNumber)
+            incidentDir.mkdir(parents=True)
+            dest = incidentDir / f"{newFilename}{extension}"
+            part.save_as(dest)
+            self._log.info(f"saved {part.filename} as {dest}")
+
+        user: IMSUser = request.user  # type: ignore[attr-defined]
+        author = user.shortNames[0]
+
+        now = DateTime.now(UTC)
+
+        entry = ReportEntry(
+            id=-1,  # will be assigned a valid ID on write to DB
+            author=author,
+            text=f"User uploaded files: {', '.join(p.filename for p in p.parts())}",
+            created=now,
+            automatic=False,
+            stricken=False,
+        )
+
+        await store.addReportEntriesToIncident(
+            eventId, incidentNumber, (entry,), author
+        )
+        return noContentResponse(request)
+
     @router.route(_unprefix(URLs.incident_reportEntry), methods=("POST",))
     async def editIncidentReportEntryResource(
         self,

src/ims/config/_urls.py

@@ -58,6 +58,7 @@ class URLs:
     incident_reportEntry: ClassVar[URL] = incident_reportEntries.child(
         "<report_entry_id>"
     )
+    incidentAttach: ClassVar[URL] = incidentNumber.child("attach")
     fieldReports: ClassVar[URL] = event.child("field_reports").child("")
     fieldReport: ClassVar[URL] = fieldReports.child("<field_report_number>")
     fieldReport_reportEntries: ClassVar[URL] = fieldReport.child("report_entries")

src/ims/element/incident/incident_template/template.xhtml

@@ -253,14 +253,25 @@
                   autofocus=""
                   onchange="reportEntryEdited()"
           />
-          <button
-                  id="report_entry_submit"
-                  type="submit"
-                  class="btn btn-default btn-sm btn-block my-1 disabled no-print"
-                  onclick="submitReportEntry()"
-          >
-            Add Entry (Control ⏎)
-          </button>
+          <div class="d-flex justify-content-between">
+            <button
+                    id="report_entry_submit"
+                    type="submit"
+                    class="btn btn-default btn-sm btn-block my-1 disabled no-print float-start"
+                    onclick="submitReportEntry()"
+            >
+              Add Entry (Control ⏎)
+            </button>
+            <!-- File attachment -->
+            <label class="input-group-text hidden" for="attach-file">Attach file</label>
+            <input type="file" multiple="" class="form-control hidden" id="attach-file" name="filename" onchange="attachFile();" />
+            <input
+                    type="button"
+                    class="btn btn-default btn-sm btn-block btn-secondary my-1 no-print"
+                    value="Attach file(s)"
+                    onclick="document.getElementById('attach-file').click();"
+            />
+          </div>
         </div>
       </div>
     </div>

src/ims/element/static/field_report.js

@@ -35,8 +35,8 @@ async function initFieldReportPage() {
         }
     });
 
-    // Updates...it's fine to ignore the returned promise here
-    requestEventSourceLock();
+    // Fire-and-forget this promise, since it tries forever to acquire a lock
+    let ignoredPromise = requestEventSourceLock();
 
     const fieldReportChannel = new BroadcastChannel(fieldReportChannelName);
     fieldReportChannel.onmessage = async function (e) {
@@ -312,40 +312,36 @@ async function editSummary() {
 
 async function makeIncident() {
     // Create the new incident
-    {
-        const incidentsURL = urlReplace(url_incidents);
+    const incidentsURL = urlReplace(url_incidents);
 
-        const authors = [];
-        if (fieldReport.report_entries?.length > 0) {
-            authors.push(fieldReport.report_entries[0].author);
-        }
-        let {resp, err} = await fetchJsonNoThrow(incidentsURL, {
-            body:{
-                "summary": fieldReport.summary,
-                "ranger_handles": authors,
-            },
-        })
-        if (err != null) {
-            disableEditing();
-            setErrorMessage(`Failed to create incident: ${err}`);
-            return;
-        }
-        fieldReport.incident = parseInt(resp.headers.get("X-IMS-Incident-Number"));
+    const authors = [];
+    if (fieldReport.report_entries?.length > 0) {
+        authors.push(fieldReport.report_entries[0].author);
     }
+    let {resp, err} = await fetchJsonNoThrow(incidentsURL, {
+        body:{
+            "summary": fieldReport.summary,
+            "ranger_handles": authors,
+        },
+    })
+    if (err != null) {
+        disableEditing();
+        setErrorMessage(`Failed to create incident: ${err}`);
+        return;
+    }
+    fieldReport.incident = parseInt(resp.headers.get("X-IMS-Incident-Number"));
 
     // Attach this FR to that new incident
-    {
-        const attachToIncidentUrl =
-            `${urlReplace(url_fieldReports)}${fieldReport.number}` +
-            `?action=attach;incident=${fieldReport.incident}`;
-        let {err} = await fetchJsonNoThrow(attachToIncidentUrl, {
-            body: {},
-        });
-        if (err != null) {
-            disableEditing();
-            setErrorMessage(`Failed to attach field report: ${err}`);
-            return;
-        }
+    const attachToIncidentUrl =
+        `${urlReplace(url_fieldReports)}${fieldReport.number}` +
+        `?action=attach;incident=${fieldReport.incident}`;
+    const {err: attachErr} = await fetchJsonNoThrow(attachToIncidentUrl, {
+        body: {},
+    });
+    if (attachErr != null) {
+        disableEditing();
+        setErrorMessage(`Failed to attach field report: ${attachErr}`);
+        return;
     }
     console.log("Created and attached to new incident " + fieldReport.incident);
     await loadAndDisplayFieldReport();

src/ims/element/static/field_reports.js

@@ -75,8 +75,9 @@ function initFieldReportsTable() {
         enableEditing();
     }
 
-    // it's ok to ignore the returned promise
-    requestEventSourceLock();
+    // Fire-and-forget this promise, since it tries forever to acquire a lock
+    let ignoredPromise = requestEventSourceLock();
+
     const fieldReportChannel = new BroadcastChannel(fieldReportChannelName);
     fieldReportChannel.onmessage = function (e) {
         if (e.data["update_all"]) {

src/ims/element/static/ims.js

@@ -188,9 +188,15 @@ async function fetchJsonNoThrow(url, init) {
     init["headers"]["Accept"] = "application/json";
     if ("body" in init) {
         init["method"] = "POST";
-        init["headers"]["Content-Type"] = "application/json";
-        if (typeof(init["body"]) !== "string") {
-            init["body"] = JSON.stringify(init["body"]);
+
+        if (init["body"].constructor.name === "FormData") {
+            // don't JSONify, don't set a Content-Type (fetch does it automatically)
+        } else {
+            // otherwise assume body is supposed to be json
+            init["headers"]["Content-Type"] = "application/json";
+            if (typeof init["body"] !== "string") {
+                init["body"] = JSON.stringify(init["body"]);
+            }
         }
     }
     let response = null;
@@ -277,6 +283,8 @@ function enable(element) {
 // Disable editing for an element
 function disableEditing() {
     disable($(".form-control"));
+    disable($("#entries-form :input"));
+    disable($("#attach-file-form :input"));
     enable($(":input[type=search]"));  // Don't disable search fields
     $(document.documentElement).addClass("no-edit");
 }
@@ -285,9 +293,18 @@ function disableEditing() {
 // Enable editing for an element
 function enableEditing() {
     enable($(".form-control"));
+    enable($("#entries-form :input"));
+    enable($("#attach-file-form :input"));
     $(document.documentElement).removeClass("no-edit");
 }
 
+function hide(element) {
+    element.addClass("hidden");
+}
+
+function unhide(element) {
+    element.removeClass("hidden");
+}
 
 // Add an error indication to a control
 function controlHasError(element) {

src/ims/element/static/incident.js

@@ -43,8 +43,8 @@ async function initIncidentPage() {
         }
     });
 
-    // Updates...it's good to ignore the returned promise here
-    requestEventSourceLock();
+    // Fire-and-forget this promise, since it tries forever to acquire a lock
+    let ignoredPromise = requestEventSourceLock();
 
     const incidentChannel = new BroadcastChannel(incidentChannelName);
     incidentChannel.onmessage = async function (e) {
@@ -188,6 +188,10 @@ async function loadAndDisplayIncident() {
     if (editingAllowed) {
         enableEditing();
     }
+
+    if (incident.number == null) {
+        hide($("#attach-file-form :input"));
+    }
 }
 
 // Do all the client-side rendering based on the state of allFieldReports.
@@ -291,12 +295,13 @@ async function loadAllFieldReports() {
     return {err: null};
 }
 
-async function loadFieldReport(fieldReportNumber, success) {
+async function loadFieldReport(fieldReportNumber) {
     if (allFieldReports === undefined) {
         return;
     }
 
-    const {resp, json, err} = await fetchJsonNoThrow(urlReplace(url_fieldReport).replace("<field_report_number>", fieldReportNumber))
+    const {resp, json, err} = await fetchJsonNoThrow(
+        urlReplace(url_fieldReport).replace("<field_report_number>", fieldReportNumber));
     if (err != null) {
         if (resp.status !== 403) {
             const message = `Failed to load field report ${fieldReportNumber} ${err}`;
@@ -781,11 +786,10 @@ async function sendEdits(edits) {
     });
 
     if (err != null) {
-        const message = "Failed to apply edit";
-        console.log(message + ": " + err);
+        const message = `Failed to apply edit: ${err}`;
         await loadAndDisplayIncident();
         setErrorMessage(message);
-        return {err: err}
+        return {err: message}
     }
 
     if (number == null) {
@@ -1035,13 +1039,11 @@ async function detachFieldReport(sender) {
 
 async function attachFieldReport() {
     if (incidentNumber == null) {
-        // Incident doesn't exist yet.  Create it and then retry.
+        // Incident doesn't exist yet.  Create it first.
         const {err} = await sendEdits({});
         if (err != null) {
             return;
         }
-        await attachFieldReport();
-        return;
     }
 
     const select = $("#attached_field_report_add");
@@ -1076,3 +1078,31 @@ async function onStrikeSuccess() {
     renderFieldReportData();
     clearErrorMessage();
 }
+
+async function attachFile() {
+    if (incidentNumber == null) {
+        // Incident doesn't exist yet.  Create it first.
+        const {err} = await sendEdits({});
+        if (err != null) {
+            return;
+        }
+    }
+    const attachFile = document.getElementById("attach-file");
+    const formData = new FormData();
+
+    for (const f of attachFile.files) {
+        formData.append("files", f);
+    }
+
+    const attachURL = urlReplace(url_incidentAttach).replace("<incident_number>", incidentNumber);
+    const {err} = await fetchJsonNoThrow(attachURL, {
+        body: formData
+    });
+    if (err != null) {
+        const message = `Failed to attach file: ${err}`;
+        setErrorMessage(message);
+        return;
+    }
+    clearErrorMessage();
+    attachFile.value = "";
+}