DO NOT MERGE - file upload progress checkin

pyproject.toml

@@ -21,6 +21,8 @@ dependencies = [
     "hyperlink==21.0.0",
     "jwcrypto==1.5.6",
     "klein==24.8.0",
+    "multipart>=1.2.1",
+    "puremagic>=1.28",
     "pymysql==1.1.1",
     "pyopenssl==25.0.0",
     "pyyaml==6.0.2",

src/ims/application/_api.py

@@ -31,17 +31,22 @@
 from functools import partial
 from json import JSONDecodeError
 from typing import Any, ClassVar, cast
+from uuid import uuid4
 
 from attrs import frozen
 from hyperlink import URL
 from klein import KleinRenderable
 from klein._app import KleinSynchronousRenderable
+from multipart import MultipartParser, parse_options_header
+from puremagic import PureError
+from puremagic import from_string as puremagic_from_string
 from twisted.internet.defer import Deferred
 from twisted.internet.error import ConnectionDone
 from twisted.logger import Logger
 from twisted.python.failure import Failure
 from twisted.web import http
 from twisted.web.iweb import IRequest
+from twisted.web.static import File
 
 from ims.auth import Authorization, NotAuthorizedError
 from ims.config import Configuration, URLs
@@ -684,6 +689,108 @@ def _cast(obj: Any) -> Any:
 
         return noContentResponse(request)
 
+    @router.route(_unprefix(URLs.incidentAttachments), methods=("POST",))
+    async def attachFileToIncident(
+        self,
+        request: IRequest,
+        event_id: str,
+        incident_number: str,
+    ) -> KleinSynchronousRenderable:
+        eventId = event_id
+        incidentNumber = int(incident_number)
+        del event_id
+        del incident_number
+
+        await self.config.authProvider.authorizeRequest(
+            request, eventId, Authorization.writeIncidents
+        )
+
+        _, options = parse_options_header(request.getHeader("Content-Type"))
+        p = MultipartParser(
+            request.content,
+            options.get("boundary", ""),
+            # Only allow one file per upload. This fits better with
+            # the REPORT_ENTRY model.
+            part_limit=1,
+            # TODO: move this to config
+            # Arbitrary 10 MB per file limit
+            partsize_limit=10 * 1024 * 1024,
+        )
+
+        parts = p.parts()
+        if len(parts) == 0:
+            # no files provided, nothing to do
+            return noContentResponse(request)
+        if len(parts) > 1:
+            return badRequestResponse(request, "Only one file is allowed per request")
+        part = parts[0]
+
+        store = self.config.store
+
+        eventsDir = self.config.attachmentsRoot / "events"
+        secretFilename = uuid4()
+        try:
+            extension = puremagic_from_string(part.raw, filename=part.filename)
+        except PureError:
+            self._log.info(f"failed to determine filetype for {part.filename}")
+            extension = ""
+        else:
+            self._log.info(f"detected file extension {extension} for {part.filename}")
+
+        incidentDir = eventsDir / eventId / "incidents" / str(incidentNumber)
+        incidentDir.mkdir(parents=True, exist_ok=True)
+        dest = incidentDir / f"{secretFilename}{extension}"
+        part.save_as(dest)
+        self._log.info(f"saved {part.filename} as {dest}")
+
+        user: IMSUser = request.user  # type: ignore[attr-defined]
+        author = user.shortNames[0]
+
+        now = DateTime.now(UTC)
+
+        entry = ReportEntry(
+            id=-1,  # will be assigned a valid ID on write to DB
+            author=author,
+            text=f"User uploaded file: {part.filename}",
+            created=now,
+            automatic=False,
+            stricken=False,
+        )
+
+        await store.addReportEntriesToIncident(
+            eventId, incidentNumber, (entry,), author
+        )
+        return noContentResponse(request)
+
+    @router.route(_unprefix(URLs.incidentAttachmentNumber), methods=("HEAD", "GET"))
+    async def retrieveIncidentAttachment(
+        self,
+        request: IRequest,
+        event_id: str,
+        incident_number: str,
+        attachment_number: str,
+    ) -> KleinSynchronousRenderable:
+        eventId = event_id
+        incidentNumber = int(incident_number)
+        del event_id
+        del incident_number
+
+        await self.config.authProvider.authorizeRequest(
+            request, eventId, Authorization.readIncidents
+        )
+
+        # TODO: will need to look through report entries to get the secret file name
+
+        folder = (
+            self.config.attachmentsRoot
+            / "events"
+            / eventId
+            / "incidents"
+            / str(incidentNumber)
+        )
+
+        return File(folder / "b982735c-3165-43b7-8bd4-6724eab141a6.pdf")
+
     @router.route(_unprefix(URLs.incident_reportEntry), methods=("POST",))
     async def editIncidentReportEntryResource(
         self,

src/ims/config/_config.py

@@ -226,6 +226,12 @@ def fromConfigFile(cls, configFile: Path | None) -> "Configuration":
         cachedResourcesRoot.mkdir(exist_ok=True)
         cls._log.info("CachedResources: {path}", path=cachedResourcesRoot)
 
+        attachmentsRoot = parser.pathFromConfig(
+            "ATTACHMENTS_ROOT", "Core", "AttachmentsRoot", serverRoot, ("attachments",)
+        )
+        attachmentsRoot.mkdir(parents=True, exist_ok=True)
+        cls._log.info("AttachmentsRoot: {path}", path=attachmentsRoot)
+
         logLevelName = parser.valueFromConfig("LOG_LEVEL", "Core", "LogLevel", "info")
         cls._log.info("LogLevel: {logLevel}", logLevel=logLevelName)
 
@@ -393,6 +399,7 @@ def fromConfigFile(cls, configFile: Path | None) -> "Configuration":
         #
 
         return cls(
+            attachmentsRoot=attachmentsRoot,
             cachedResourcesRoot=cachedResourcesRoot,
             configFile=configFile,
             configRoot=configRoot,
@@ -412,6 +419,7 @@ def fromConfigFile(cls, configFile: Path | None) -> "Configuration":
             tokenLifetime=tokenLifetime,
         )
 
+    attachmentsRoot: Path
     cachedResourcesRoot: Path
     configFile: Path | None
     configRoot: Path
@@ -467,6 +475,7 @@ def __str__(self) -> str:
             f"Core.Port: {self.port}\n"
             f"\n"
             f"Core.ServerRoot: {self.serverRoot}\n"
+            f"Core.AttachmentsRoot: {self.attachmentsRoot}\n"
             f"Core.ConfigRoot: {self.configRoot}\n"
             f"Core.DataRoot: {self.dataRoot}\n"
             f"Core.CachedResources: {self.cachedResourcesRoot}\n"

src/ims/config/_urls.py

@@ -58,6 +58,10 @@ class URLs:
     incident_reportEntry: ClassVar[URL] = incident_reportEntries.child(
         "<report_entry_id>"
     )
+    incidentAttachments: ClassVar[URL] = incidentNumber.child("attachments").child("")
+    incidentAttachmentNumber: ClassVar[URL] = incidentAttachments.child(
+        "<attachment_number>"
+    )
     fieldReports: ClassVar[URL] = event.child("field_reports").child("")
     fieldReport: ClassVar[URL] = fieldReports.child("<field_report_number>")
     fieldReport_reportEntries: ClassVar[URL] = fieldReport.child("report_entries")

src/ims/config/test/test_config.py

@@ -749,6 +749,7 @@ def test_str(self) -> None:
             f"Core.Port: {config.port}\n"
             f"\n"
             f"Core.ServerRoot: {config.serverRoot}\n"
+            f"Core.AttachmentsRoot: {config.attachmentsRoot}\n"
             f"Core.ConfigRoot: {config.configRoot}\n"
             f"Core.DataRoot: {config.dataRoot}\n"
             f"Core.CachedResources: {config.cachedResourcesRoot}\n"

src/ims/element/incident/incident_template/template.xhtml

@@ -253,14 +253,25 @@
                   autofocus=""
                   onchange="reportEntryEdited()"
           />
-          <button
-                  id="report_entry_submit"
-                  type="submit"
-                  class="btn btn-default btn-sm btn-block my-1 disabled no-print"
-                  onclick="submitReportEntry()"
-          >
-            Add Entry (Control ⏎)
-          </button>
+          <div class="d-flex justify-content-between">
+            <button
+                    id="report_entry_submit"
+                    type="submit"
+                    class="btn btn-default btn-sm btn-block my-1 disabled no-print float-start"
+                    onclick="submitReportEntry()"
+            >
+              Add Entry (Control ⏎)
+            </button>
+            <!-- File attachment -->
+            <label class="input-group-text hidden" for="attach-file">Attach file</label>
+            <input type="file" class="form-control hidden" id="attach-file" name="filename" onchange="attachFile();" />
+            <input
+                    type="button"
+                    class="btn btn-default btn-sm btn-block btn-secondary my-1 no-print"
+                    value="Attach file"
+                    onclick="document.getElementById('attach-file').click();"
+            />
+          </div>
         </div>
       </div>
     </div>

src/ims/element/static/ims.js

@@ -188,9 +188,15 @@ async function fetchJsonNoThrow(url, init) {
     init["headers"]["Accept"] = "application/json";
     if ("body" in init) {
         init["method"] = "POST";
-        init["headers"]["Content-Type"] = "application/json";
-        if (typeof(init["body"]) !== "string") {
-            init["body"] = JSON.stringify(init["body"]);
+
+        if (init["body"].constructor.name === "FormData") {
+            // don't JSONify, don't set a Content-Type (fetch does it automatically)
+        } else {
+            // otherwise assume body is supposed to be json
+            init["headers"]["Content-Type"] = "application/json";
+            if (typeof init["body"] !== "string") {
+                init["body"] = JSON.stringify(init["body"]);
+            }
         }
     }
     let response = null;
@@ -277,6 +283,8 @@ function enable(element) {
 // Disable editing for an element
 function disableEditing() {
     disable($(".form-control"));
+    disable($("#entries-form :input"));
+    disable($("#attach-file-form :input"));
     enable($(":input[type=search]"));  // Don't disable search fields
     $(document.documentElement).addClass("no-edit");
 }
@@ -285,9 +293,18 @@ function disableEditing() {
 // Enable editing for an element
 function enableEditing() {
     enable($(".form-control"));
+    enable($("#entries-form :input"));
+    enable($("#attach-file-form :input"));
     $(document.documentElement).removeClass("no-edit");
 }
 
+function hide(element) {
+    element.addClass("hidden");
+}
+
+function unhide(element) {
+    element.removeClass("hidden");
+}
 
 // Add an error indication to a control
 function controlHasError(element) {

src/ims/element/static/incident.js

@@ -188,6 +188,10 @@ async function loadAndDisplayIncident() {
     if (editingAllowed) {
         enableEditing();
     }
+
+    if (incident.number == null) {
+        hide($("#attach-file-form :input"));
+    }
 }
 
 // Do all the client-side rendering based on the state of allFieldReports.
@@ -1074,3 +1078,32 @@ async function onStrikeSuccess() {
     renderFieldReportData();
     clearErrorMessage();
 }
+
+async function attachFile() {
+    if (incidentNumber == null) {
+        // Incident doesn't exist yet.  Create it first.
+        const {err} = await sendEdits({});
+        if (err != null) {
+            return;
+        }
+    }
+    const attachFile = document.getElementById("attach-file");
+    const formData = new FormData();
+
+    for (const f of attachFile.files) {
+        formData.append("files", f);
+    }
+
+    const attachURL = urlReplace(url_incidentAttachments).replace("<incident_number>", incidentNumber);
+    const {err} = await fetchJsonNoThrow(attachURL, {
+        body: formData
+    });
+    if (err != null) {
+        const message = `Failed to attach file: ${err}`;
+        setErrorMessage(message);
+        return;
+    }
+    clearErrorMessage();
+    attachFile.value = "";
+    await loadAndDisplayIncident();
+}

src/ims/store/_db.py

@@ -18,6 +18,7 @@
 Incident Management System database tooling.
 """
 
+import re
 from abc import abstractmethod
 from collections import defaultdict
 from collections.abc import Callable, Iterable, Iterator, Mapping
@@ -366,6 +367,13 @@ async def createEvent(self, event: Event) -> None:
         """
         See :meth:`IMSDataStore.createEvent`.
         """
+
+        eventIdPattern = r"^[\w-]+$"
+        if not re.search(eventIdPattern, event.id):
+            raise ValueError(
+                f"wanted EventID to match '{eventIdPattern}', got '{event.id}'"
+            )
+
         await self.runOperation(self.query.createEvent, {"eventID": event.id})
 
         self._log.info(

src/ims/store/mysql/_store.py

@@ -100,7 +100,7 @@ class DataStore(DatabaseStore):
 
     _log: ClassVar[Logger] = Logger()
 
-    schemaVersion: ClassVar[int] = 9
+    schemaVersion: ClassVar[int] = 10
     schemaBasePath: ClassVar[Path] = Path(__file__).parent / "schema"
     sqlFileExtension: ClassVar[str] = "mysql"
 

src/ims/store/mysql/schema/10-from-9.mysql

@@ -0,0 +1,9 @@
+/* Add support for file attachments */
+
+alter table REPORT_ENTRY
+    add column ATTACHED_FILE varchar(128)
+;
+
+/* Update schema version */
+
+update `SCHEMA_INFO` set `VERSION` = 10;