From 74143bfebcdd47b4bb0c8015f7fb19fba299a47d Mon Sep 17 00:00:00 2001 From: David Mears Date: Wed, 15 Jan 2025 20:32:10 +0000 Subject: [PATCH 01/12] Add (non-functional) zip buttons to download page --- .../contents/downloads/Downloads.tsx | 10 +++++-- .../contents/downloads/orderly/Artefacts.tsx | 4 ++- .../contents/downloads/orderly/OtherFiles.tsx | 28 +++++++++++-------- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/app/src/app/components/contents/downloads/Downloads.tsx b/app/src/app/components/contents/downloads/Downloads.tsx index acf96b72..d89e5651 100644 --- a/app/src/app/components/contents/downloads/Downloads.tsx +++ b/app/src/app/components/contents/downloads/Downloads.tsx @@ -4,6 +4,7 @@ import { PacketHeader } from "../packets"; import { Card, CardContent } from "../../Base/Card"; import { FileRow } from "./FileRow"; import { OrderlyDownloads } from "./orderly/OrderlyDownloads"; +import { DownloadAllFilesButton } from "./orderly/DownloadAllFilesButton"; export const Downloads = () => { const { packetId, packetName } = useParams(); @@ -11,10 +12,15 @@ export const Downloads = () => { const packetIsFromOrderly = !!packet?.custom?.orderly; return ( - <> +
{packet && ( <> +
+ + + +
{packetIsFromOrderly && } {!packetIsFromOrderly && ( <> @@ -34,6 +40,6 @@ export const Downloads = () => { )} )} - +
); }; diff --git a/app/src/app/components/contents/downloads/orderly/Artefacts.tsx b/app/src/app/components/contents/downloads/orderly/Artefacts.tsx index 107d8f0e..d358a7f2 100644 --- a/app/src/app/components/contents/downloads/orderly/Artefacts.tsx +++ b/app/src/app/components/contents/downloads/orderly/Artefacts.tsx @@ -1,6 +1,7 @@ import { Card, CardContent, CardHeader } from "../../../Base/Card"; import { FileRow } from "../FileRow"; import { Artefact } from "../../../../../types"; +import { FileGroupDownloadButton } from "./FileGroupDownloadButton"; interface ArtefactsProps { artefacts: Artefact[]; @@ -12,8 +13,9 @@ export const Artefacts = ({ artefacts }: ArtefactsProps) => { {artefacts.map((artefact, key) => (
  • - +

    {artefact.description}

    +
      diff --git a/app/src/app/components/contents/downloads/orderly/OtherFiles.tsx b/app/src/app/components/contents/downloads/orderly/OtherFiles.tsx index a4794da0..bde17227 100644 --- a/app/src/app/components/contents/downloads/orderly/OtherFiles.tsx +++ b/app/src/app/components/contents/downloads/orderly/OtherFiles.tsx @@ -1,6 +1,7 @@ import { Card, CardContent } from "../../../Base/Card"; import { FileRow } from "../FileRow"; import { InputFiles, InputFileType } from "../../../../../types"; +import { FileGroupDownloadButton } from "./FileGroupDownloadButton"; // 'Other files' are any files to display other than artefact groups. @@ -10,16 +11,21 @@ interface OtherFilesProps { export const OtherFiles = ({ inputFiles }: OtherFilesProps) => { return ( - - -
        - {inputFiles.map((input, index) => ( -
      • - -
        • - ))} -
      -       -       +
      + + + + + +
        + {inputFiles.map((input, index) => ( +
      • + +
        • + ))} +
      +       +       +
      ); }; From 77d8b11fc5d62f4519020e1eed6954ffc6f5bc6c Mon Sep 17 00:00:00 2001 From: David Mears Date: Wed, 15 Jan 2025 20:43:20 +0000 Subject: [PATCH 02/12] Move download-all-files button --- .../app/components/contents/downloads/Downloads.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/src/app/components/contents/downloads/Downloads.tsx b/app/src/app/components/contents/downloads/Downloads.tsx index d89e5651..a3c55bca 100644 --- a/app/src/app/components/contents/downloads/Downloads.tsx +++ b/app/src/app/components/contents/downloads/Downloads.tsx @@ -12,12 +12,12 @@ export const Downloads = () => { const packetIsFromOrderly = !!packet?.custom?.orderly; return ( -
      + <> {packet && ( <> - -
      - +
      + +
      @@ -40,6 +40,6 @@ export const Downloads = () => { )} )} -
      + ); }; From b80eabd337c2a380771cb75f5569aee42cb4eec2 Mon Sep 17 00:00:00 2001 From: David Mears Date: Wed, 15 Jan 2025 20:56:08 +0000 Subject: [PATCH 03/12] Include missing files forgotten from previous commits --- .../downloads/orderly/DownloadAllFilesButton.tsx | 10 ++++++++++ .../downloads/orderly/FileGroupDownloadButton.tsx | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 app/src/app/components/contents/downloads/orderly/DownloadAllFilesButton.tsx create mode 100644 app/src/app/components/contents/downloads/orderly/FileGroupDownloadButton.tsx diff --git a/app/src/app/components/contents/downloads/orderly/DownloadAllFilesButton.tsx b/app/src/app/components/contents/downloads/orderly/DownloadAllFilesButton.tsx new file mode 100644 index 00000000..16b33aec --- /dev/null +++ b/app/src/app/components/contents/downloads/orderly/DownloadAllFilesButton.tsx @@ -0,0 +1,10 @@ +import { Button } from "../../../Base/Button"; +import { FolderDown } from "lucide-react"; + +export const DownloadAllFilesButton = () => { + return ( + + ); +}; diff --git a/app/src/app/components/contents/downloads/orderly/FileGroupDownloadButton.tsx b/app/src/app/components/contents/downloads/orderly/FileGroupDownloadButton.tsx new file mode 100644 index 00000000..958f2680 --- /dev/null +++ b/app/src/app/components/contents/downloads/orderly/FileGroupDownloadButton.tsx @@ -0,0 +1,10 @@ +import { Button } from "../../../Base/Button"; +import { FolderDown } from "lucide-react"; + +export const FileGroupDownloadButton = () => { + return ( + + ); +}; From 98a38de93caa649e1c9e13a619e9e2bcc8547f78 Mon Sep 17 00:00:00 2001 From: David Mears Date: Tue, 21 Jan 2025 17:29:47 +0000 Subject: [PATCH 04/12] Re-jiggle style and layout to allow for another type of zip download --- .../contents/downloads/orderly/Artefacts.tsx | 48 +++++++++++-------- .../orderly/DownloadAllFilesButton.tsx | 2 +- .../orderly/FileGroupDownloadButton.tsx | 2 +- .../downloads/orderly/OrderlyDownloads.tsx | 40 +++++++++------- .../contents/downloads/orderly/OtherFiles.tsx | 8 ++-- 5 files changed, 55 insertions(+), 45 deletions(-) diff --git a/app/src/app/components/contents/downloads/orderly/Artefacts.tsx b/app/src/app/components/contents/downloads/orderly/Artefacts.tsx index d358a7f2..ab8ab46c 100644 --- a/app/src/app/components/contents/downloads/orderly/Artefacts.tsx +++ b/app/src/app/components/contents/downloads/orderly/Artefacts.tsx @@ -2,6 +2,7 @@ import { Card, CardContent, CardHeader } from "../../../Base/Card"; import { FileRow } from "../FileRow"; import { Artefact } from "../../../../../types"; import { FileGroupDownloadButton } from "./FileGroupDownloadButton"; +import { DownloadAllArtefactsButton } from "./DownloadAllArtefactsButton"; interface ArtefactsProps { artefacts: Artefact[]; @@ -9,26 +10,31 @@ interface ArtefactsProps { export const Artefacts = ({ artefacts }: ArtefactsProps) => { return ( -
        - {artefacts.map((artefact, key) => ( -
      • - - -

        {artefact.description}

        - -         - -
          - {artefact.paths.map((path, index) => ( -
        • - -
          • - ))} -
        -         -         -
        • - ))} -
      + <> + + + +
        + {artefacts.map((artefact, key) => ( +
      • + + +

        {artefact.description}

        + +         + +
          + {artefact.paths.map((path, index) => ( +
        • + +
          • + ))} +
        +         +         +
        • + ))} +
      + ); }; diff --git a/app/src/app/components/contents/downloads/orderly/DownloadAllFilesButton.tsx b/app/src/app/components/contents/downloads/orderly/DownloadAllFilesButton.tsx index 16b33aec..a0ea008f 100644 --- a/app/src/app/components/contents/downloads/orderly/DownloadAllFilesButton.tsx +++ b/app/src/app/components/contents/downloads/orderly/DownloadAllFilesButton.tsx @@ -4,7 +4,7 @@ import { FolderDown } from "lucide-react"; export const DownloadAllFilesButton = () => { return ( ); }; diff --git a/app/src/app/components/contents/downloads/orderly/FileGroupDownloadButton.tsx b/app/src/app/components/contents/downloads/orderly/FileGroupDownloadButton.tsx index 958f2680..cda831df 100644 --- a/app/src/app/components/contents/downloads/orderly/FileGroupDownloadButton.tsx +++ b/app/src/app/components/contents/downloads/orderly/FileGroupDownloadButton.tsx @@ -3,7 +3,7 @@ import { FolderDown } from "lucide-react"; export const FileGroupDownloadButton = () => { return ( - ); diff --git a/app/src/app/components/contents/downloads/orderly/OrderlyDownloads.tsx b/app/src/app/components/contents/downloads/orderly/OrderlyDownloads.tsx index 11799fab..38daf4f9 100644 --- a/app/src/app/components/contents/downloads/orderly/OrderlyDownloads.tsx +++ b/app/src/app/components/contents/downloads/orderly/OrderlyDownloads.tsx @@ -13,24 +13,28 @@ export const OrderlyDownloads = () => { if (!!artefacts?.length && !!inputFiles?.length) { return ( - - - -

      Artefacts

      -       - - - -       - - -

      Other files

      -       - - - -       -       +
      + + + +

      Artefacts

      +       + + + +       +
      + + +

      Other files

      +       + + + +       +
      +       +
      ); } else if (artefacts?.length) { return ( diff --git a/app/src/app/components/contents/downloads/orderly/OtherFiles.tsx b/app/src/app/components/contents/downloads/orderly/OtherFiles.tsx index bde17227..361a210a 100644 --- a/app/src/app/components/contents/downloads/orderly/OtherFiles.tsx +++ b/app/src/app/components/contents/downloads/orderly/OtherFiles.tsx @@ -1,7 +1,7 @@ import { Card, CardContent } from "../../../Base/Card"; import { FileRow } from "../FileRow"; import { InputFiles, InputFileType } from "../../../../../types"; -import { FileGroupDownloadButton } from "./FileGroupDownloadButton"; +import { DownloadAllOtherFilesButton } from "./DownloadAllOtherFilesButton"; // 'Other files' are any files to display other than artefact groups. @@ -11,9 +11,9 @@ interface OtherFilesProps { export const OtherFiles = ({ inputFiles }: OtherFilesProps) => { return ( -
      - - +
      + + From efc0ddd636f97b1ce7faf6343d43fd6d0af5d274 Mon Sep 17 00:00:00 2001 From: David Mears Date: Tue, 21 Jan 2025 17:33:55 +0000 Subject: [PATCH 05/12] Add files forgotten from prev commit --- .../downloads/orderly/DownloadAllArtefactsButton.tsx | 9 +++++++++ .../downloads/orderly/DownloadAllOtherFilesButton.tsx | 9 +++++++++ 2 files changed, 18 insertions(+) create mode 100644 app/src/app/components/contents/downloads/orderly/DownloadAllArtefactsButton.tsx create mode 100644 app/src/app/components/contents/downloads/orderly/DownloadAllOtherFilesButton.tsx diff --git a/app/src/app/components/contents/downloads/orderly/DownloadAllArtefactsButton.tsx b/app/src/app/components/contents/downloads/orderly/DownloadAllArtefactsButton.tsx new file mode 100644 index 00000000..18262cad --- /dev/null +++ b/app/src/app/components/contents/downloads/orderly/DownloadAllArtefactsButton.tsx @@ -0,0 +1,9 @@ +import { Button } from "../../../Base/Button"; + +export const DownloadAllArtefactsButton = () => { + return ( + + ); +}; diff --git a/app/src/app/components/contents/downloads/orderly/DownloadAllOtherFilesButton.tsx b/app/src/app/components/contents/downloads/orderly/DownloadAllOtherFilesButton.tsx new file mode 100644 index 00000000..50841362 --- /dev/null +++ b/app/src/app/components/contents/downloads/orderly/DownloadAllOtherFilesButton.tsx @@ -0,0 +1,9 @@ +import { Button } from "../../../Base/Button"; + +export const DownloadAllOtherFilesButton = () => { + return ( + + ); +}; From cb04f6cd2b1d6088c93656d1727838b2613edeee Mon Sep 17 00:00:00 2001 From: David Mears Date: Tue, 28 Jan 2025 19:40:10 +0000 Subject: [PATCH 06/12] wipzip --- .../main/kotlin/packit/config/WebConfig.kt | 24 ++++++++++++++ .../packit/controllers/PacketController.kt | 31 ++++++++++++++++++- .../main/kotlin/packit/service/ZipService.kt | 30 ++++++++++++++++++ .../components/contents/downloads/FileRow.tsx | 4 +-- .../orderly/DownloadAllArtefactsButton.tsx | 24 +++++++++++++- .../orderly/DownloadAllFilesButton.tsx | 16 +++++++++- .../contents/packets/PacketDetails.tsx | 1 + app/src/lib/download.ts | 4 +++ 8 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 api/app/src/main/kotlin/packit/config/WebConfig.kt create mode 100644 api/app/src/main/kotlin/packit/service/ZipService.kt diff --git a/api/app/src/main/kotlin/packit/config/WebConfig.kt b/api/app/src/main/kotlin/packit/config/WebConfig.kt new file mode 100644 index 00000000..1e0b48ad --- /dev/null +++ b/api/app/src/main/kotlin/packit/config/WebConfig.kt @@ -0,0 +1,24 @@ +package packit.config + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.context.annotation.Configuration +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor +import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer +import org.springframework.web.servlet.config.annotation.InterceptorRegistry +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer + +@Configuration +class WebConfig : WebMvcConfigurer { + // Configure the thread pool for async requests. Explicit config is 'highly recommended'. See: + // https://docs.spring.io/spring-framework/docs/4.3.12.RELEASE/spring-framework-reference/html/mvc.html#mvc-ann-async-configuration-spring-mvc + override fun configureAsyncSupport(configurer: AsyncSupportConfigurer) { + val executor = ThreadPoolTaskExecutor() + executor.corePoolSize = 10 + executor.maxPoolSize = 50 + executor.queueCapacity = 100 + executor.setThreadNamePrefix("Async-") + executor.initialize() + configurer.setTaskExecutor(executor) + configurer.setDefaultTimeout(60000) + } +} \ No newline at end of file diff --git a/api/app/src/main/kotlin/packit/controllers/PacketController.kt b/api/app/src/main/kotlin/packit/controllers/PacketController.kt index 10693212..17de6d62 100644 --- a/api/app/src/main/kotlin/packit/controllers/PacketController.kt +++ b/api/app/src/main/kotlin/packit/controllers/PacketController.kt @@ -1,10 +1,12 @@ package packit.controllers +import org.springframework.http.MediaType import org.springframework.core.io.ByteArrayResource import org.springframework.data.domain.Page import org.springframework.http.ResponseEntity import org.springframework.security.access.prepost.PreAuthorize import org.springframework.web.bind.annotation.* +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody import packit.model.PacketMetadata import packit.model.PageablePayload import packit.model.dto.PacketDto @@ -12,12 +14,14 @@ import packit.model.dto.PacketGroupSummary import packit.model.toDto import packit.service.PacketGroupService import packit.service.PacketService +import packit.service.ZipService @RestController @RequestMapping("/packets") class PacketController( private val packetService: PacketService, - private val packetGroupService: PacketGroupService + private val packetGroupService: PacketGroupService, + private val zipService: ZipService, ) { @GetMapping @@ -79,4 +83,29 @@ class PacketController( .headers(response.second) .body(response.first) } + + @GetMapping("/{name}/{id}/zip") + @PreAuthorize("@authz.canReadPacket(#root, #id, #name)") + // TODO: Verify authorization for accessing specific files: whether user has access to all files, or artefacts only. + fun downloadZip( + @PathVariable id: String, + @PathVariable name: String, + @RequestParam(required = true) hashes: List, + @RequestParam(required = true) filenames: List, + ): ResponseEntity // Note: when using this option it is highly recommended to configure explicitly the TaskExecutor used in Spring MVC for executing asynchronous requests. Both the MVC Java config and the MVC namespaces provide options to configure asynchronous handling. If not using those, an application can set the taskExecutor property of RequestMappingHandlerAdapter. + { + val filesWithNames = hashes.mapIndexed { index, hash -> + val fileResource = packetService.getFileByHash(hash, false, hash).first + fileResource to filenames[index] + } + + val streamingResponseBody = StreamingResponseBody { outputStream -> + zipService.zipByteArraysToOutputStream(filesWithNames, outputStream) + } + + return ResponseEntity + .ok() + .header("Content-Disposition", "attachment; filename=\"${name}.zip\"") + .body(streamingResponseBody) + } } diff --git a/api/app/src/main/kotlin/packit/service/ZipService.kt b/api/app/src/main/kotlin/packit/service/ZipService.kt new file mode 100644 index 00000000..d04ba6a5 --- /dev/null +++ b/api/app/src/main/kotlin/packit/service/ZipService.kt @@ -0,0 +1,30 @@ +package packit.service + +import org.springframework.core.io.ByteArrayResource +import org.springframework.stereotype.Service +import java.io.OutputStream +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +interface ZipService { + fun zipByteArraysToOutputStream(resources: List>, output: OutputStream) +} + +@Service +class BaseZipService : ZipService { + override fun zipByteArraysToOutputStream(resources: List>, output: OutputStream) { + ZipOutputStream(output).use { zipOutputStream -> + resources.forEach { (resource, filename) -> + try { + val zipEntry = ZipEntry(filename) + zipOutputStream.putNextEntry(zipEntry) + resource.inputStream.use { it.copyTo(zipOutputStream) } + zipOutputStream.closeEntry() + } catch (e: Exception) { + // Handle exception (e.g., log the error) + e.printStackTrace() + } + } + } + } +} \ No newline at end of file diff --git a/app/src/app/components/contents/downloads/FileRow.tsx b/app/src/app/components/contents/downloads/FileRow.tsx index 99613ae0..6c8bd9e3 100644 --- a/app/src/app/components/contents/downloads/FileRow.tsx +++ b/app/src/app/components/contents/downloads/FileRow.tsx @@ -10,9 +10,9 @@ interface FileRowProps { export const FileRow = ({ path, sharedResource }: FileRowProps) => { const { packet } = usePacketOutletContext(); - const file = packet?.files.filter((file) => { + const file = packet?.files.find((file) => { return file.path === path.replace("//", "/"); - })[0]; + }); const fileName = path.split("/").pop(); if (!file || !packet) return null; diff --git a/app/src/app/components/contents/downloads/orderly/DownloadAllArtefactsButton.tsx b/app/src/app/components/contents/downloads/orderly/DownloadAllArtefactsButton.tsx index 18262cad..bbaa5a4f 100644 --- a/app/src/app/components/contents/downloads/orderly/DownloadAllArtefactsButton.tsx +++ b/app/src/app/components/contents/downloads/orderly/DownloadAllArtefactsButton.tsx @@ -1,8 +1,30 @@ import { Button } from "../../../Base/Button"; +import { useParams } from "react-router-dom"; +import { usePacketOutletContext } from "../../../main/PacketOutlet"; +import { baseZipUrl, download } from "../../../../../lib/download"; export const DownloadAllArtefactsButton = () => { + const { packetId, packetName } = useParams(); + const { packet } = usePacketOutletContext(); + + const artefactsFiles = packet?.custom?.orderly.artefacts.flatMap((art) => { + return art.paths.map((path) => { + return packet?.files.find((file) => file.path === path.replace("//", "/")); + }); + }); + + const downloadAllArtefacts = async () => { + if (!packet || !packetId || !packetName || !artefactsFiles || artefactsFiles?.some((file) => file === undefined)) { + throw new Error("Error retrieving artefact files"); + } + const hashes = artefactsFiles.map((file) => file!.hash); + const filenames = artefactsFiles.map((file) => file!.path); + const url = `${baseZipUrl(packetName, packetId)}?hashes=${hashes?.join(",")}&filenames=${filenames?.join(",")}`; + await download(url, `${packetName}_artefacts_${packetId}.zip`); + }; + return ( - ); diff --git a/app/src/app/components/contents/downloads/orderly/DownloadAllFilesButton.tsx b/app/src/app/components/contents/downloads/orderly/DownloadAllFilesButton.tsx index a0ea008f..fda20a8d 100644 --- a/app/src/app/components/contents/downloads/orderly/DownloadAllFilesButton.tsx +++ b/app/src/app/components/contents/downloads/orderly/DownloadAllFilesButton.tsx @@ -1,9 +1,23 @@ import { Button } from "../../../Base/Button"; import { FolderDown } from "lucide-react"; +import { baseZipUrl, download } from "../../../../../lib/download"; +import { useParams } from "react-router-dom"; +import { usePacketOutletContext } from "../../../main/PacketOutlet"; export const DownloadAllFilesButton = () => { + const { packetId, packetName } = useParams(); + const { packet } = usePacketOutletContext(); + + const downloadAllFiles = async () => { + if (!packet || !packetId || !packetName || !packet.files) return; + const hashes = packet.files.map((file) => file.hash); + const filenames = packet.files.map((file) => file.path); + const url = `${baseZipUrl(packetName, packetId)}?hashes=${hashes?.join(",")}&filenames=${filenames?.join(",")}`; + await download(url, `${packetName}_${packetId}.zip`); + }; + return ( - ); diff --git a/app/src/app/components/contents/packets/PacketDetails.tsx b/app/src/app/components/contents/packets/PacketDetails.tsx index 0cf0b787..bd90bff1 100644 --- a/app/src/app/components/contents/packets/PacketDetails.tsx +++ b/app/src/app/components/contents/packets/PacketDetails.tsx @@ -9,6 +9,7 @@ export const PacketDetails = () => { const { packetId, packetName } = useParams(); const { packet } = usePacketOutletContext(); const longDescription = packet?.custom?.orderly.description.long; + return ( <> diff --git a/app/src/lib/download.ts b/app/src/lib/download.ts index 802e15b1..ac70e70f 100644 --- a/app/src/lib/download.ts +++ b/app/src/lib/download.ts @@ -1,4 +1,5 @@ import { getAuthHeader } from "./auth/getAuthHeader"; +import appConfig from "../config/appConfig"; export const getFileObjectUrl = async (url: string, filename: string) => { const headers = getAuthHeader(); @@ -29,3 +30,6 @@ export const download = async (url: string, filename: string) => { document.body.removeChild(fileLink); window.URL.revokeObjectURL(fileUrl); }; + +export const baseZipUrl = (packetName: string, packetId: string) => + `${appConfig.apiUrl()}/packets/${packetName}/${packetId}/zip`; From b6d80a14a3cc95ce4142ffa934157ef717c63d0e Mon Sep 17 00:00:00 2001 From: David Mears Date: Tue, 28 Jan 2025 20:18:45 +0000 Subject: [PATCH 07/12] Add functionality into all group download buttons --- .../contents/downloads/orderly/Artefacts.tsx | 44 +++++++++++-------- .../orderly/DownloadAllArtefactsButton.tsx | 9 ++-- .../orderly/DownloadAllOtherFilesButton.tsx | 23 +++++++++- .../orderly/FileGroupDownloadButton.tsx | 26 ++++++++++- 4 files changed, 75 insertions(+), 27 deletions(-) diff --git a/app/src/app/components/contents/downloads/orderly/Artefacts.tsx b/app/src/app/components/contents/downloads/orderly/Artefacts.tsx index 1d77158f..f4eb1f30 100644 --- a/app/src/app/components/contents/downloads/orderly/Artefacts.tsx +++ b/app/src/app/components/contents/downloads/orderly/Artefacts.tsx @@ -23,25 +23,31 @@ export const Artefacts = ({ artefacts }: ArtefactsProps) => {
        - {artefacts.map((artefact, key) => ( -
      • - - -

        {artefact.description}

        - -         - -
          - {filesForArtefact(artefact).map((file, index) => ( -
        • - -
          • - ))} -
        -         -         -
        • - ))} + {artefacts.map((artefact, key) => { + const files = filesForArtefact(artefact); + return ( +
      • + + +

        {artefact.description}

        + +         + +
          + {files.map((file, index) => ( +
        • + +
          • + ))} +
        +         +         +
        • + ); + })}
      ); diff --git a/app/src/app/components/contents/downloads/orderly/DownloadAllArtefactsButton.tsx b/app/src/app/components/contents/downloads/orderly/DownloadAllArtefactsButton.tsx index bbaa5a4f..63f98f57 100644 --- a/app/src/app/components/contents/downloads/orderly/DownloadAllArtefactsButton.tsx +++ b/app/src/app/components/contents/downloads/orderly/DownloadAllArtefactsButton.tsx @@ -2,23 +2,22 @@ import { Button } from "../../../Base/Button"; import { useParams } from "react-router-dom"; import { usePacketOutletContext } from "../../../main/PacketOutlet"; import { baseZipUrl, download } from "../../../../../lib/download"; +import { getFileByPath } from "../utils/getFileByPath"; export const DownloadAllArtefactsButton = () => { const { packetId, packetName } = useParams(); const { packet } = usePacketOutletContext(); const artefactsFiles = packet?.custom?.orderly.artefacts.flatMap((art) => { - return art.paths.map((path) => { - return packet?.files.find((file) => file.path === path.replace("//", "/")); - }); + return art.paths.map((path) => getFileByPath(path, packet)); }); const downloadAllArtefacts = async () => { if (!packet || !packetId || !packetName || !artefactsFiles || artefactsFiles?.some((file) => file === undefined)) { throw new Error("Error retrieving artefact files"); } - const hashes = artefactsFiles.map((file) => file!.hash); - const filenames = artefactsFiles.map((file) => file!.path); + const hashes = artefactsFiles.map((file) => file?.hash); + const filenames = artefactsFiles.map((file) => file?.path); const url = `${baseZipUrl(packetName, packetId)}?hashes=${hashes?.join(",")}&filenames=${filenames?.join(",")}`; await download(url, `${packetName}_artefacts_${packetId}.zip`); }; diff --git a/app/src/app/components/contents/downloads/orderly/DownloadAllOtherFilesButton.tsx b/app/src/app/components/contents/downloads/orderly/DownloadAllOtherFilesButton.tsx index 50841362..335e8fa7 100644 --- a/app/src/app/components/contents/downloads/orderly/DownloadAllOtherFilesButton.tsx +++ b/app/src/app/components/contents/downloads/orderly/DownloadAllOtherFilesButton.tsx @@ -1,8 +1,29 @@ import { Button } from "../../../Base/Button"; +import { useParams } from "react-router-dom"; +import { usePacketOutletContext } from "../../../main/PacketOutlet"; +import { baseZipUrl, download } from "../../../../../lib/download"; +import { getFileByPath } from "../utils/getFileByPath"; export const DownloadAllOtherFilesButton = () => { + const { packetId, packetName } = useParams(); + const { packet } = usePacketOutletContext(); + + const downloadFiles = async () => { + if (!packet || !packetId || !packetName) return; + const artefactFilePaths = packet.custom?.orderly.artefacts.flatMap((art) => { + // Route through getFileByPath for "//" substitution + return art.paths.map((path) => getFileByPath(path, packet)?.path); + }); + if (!artefactFilePaths) return; + const files = packet.files.filter((file) => !artefactFilePaths.includes(file.path)); + const hashes = files.map((file) => file.hash); + const filenames = files.map((file) => file.path); + const url = `${baseZipUrl(packetName, packetId)}?hashes=${hashes?.join(",")}&filenames=${filenames?.join(",")}`; + await download(url, `${packetName}_other_resources_${packetId}.zip`); + }; + return ( - ); diff --git a/app/src/app/components/contents/downloads/orderly/FileGroupDownloadButton.tsx b/app/src/app/components/contents/downloads/orderly/FileGroupDownloadButton.tsx index cda831df..04d160d4 100644 --- a/app/src/app/components/contents/downloads/orderly/FileGroupDownloadButton.tsx +++ b/app/src/app/components/contents/downloads/orderly/FileGroupDownloadButton.tsx @@ -1,9 +1,31 @@ +import { useParams } from "react-router-dom"; +import { baseZipUrl, download } from "../../../../../lib/download"; import { Button } from "../../../Base/Button"; import { FolderDown } from "lucide-react"; +import { usePacketOutletContext } from "../../../main/PacketOutlet"; +import { FileMetadata } from "../../../../../types"; + +interface FileGroupDownloadButtonProps { + files: FileMetadata[]; + zipName: string; +} + +export const FileGroupDownloadButton = ({ files, zipName }: FileGroupDownloadButtonProps) => { + const { packetId, packetName } = useParams(); + const { packet } = usePacketOutletContext(); + + const downloadFiles = async () => { + if (!packet || !packetId || !packetName || !files) { + return; + } + const hashes = files.map((file) => file.hash); + const filenames = files.map((file) => file.path); + const url = `${baseZipUrl(packetName, packetId)}?hashes=${hashes?.join(",")}&filenames=${filenames?.join(",")}`; + await download(url, zipName); + }; -export const FileGroupDownloadButton = () => { return ( - ); From b97e918446f40f4430de7b01aee56e1b94a0c507 Mon Sep 17 00:00:00 2001 From: David Mears Date: Wed, 29 Jan 2025 12:26:28 +0000 Subject: [PATCH 08/12] Formatting etc --- .../main/kotlin/packit/config/WebConfig.kt | 2 -- .../packit/controllers/PacketController.kt | 2 +- .../unit/controllers/PacketControllerTest.kt | 12 ++++---- .../contents/downloads/orderly/OtherFiles.tsx | 28 +++++++++---------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/api/app/src/main/kotlin/packit/config/WebConfig.kt b/api/app/src/main/kotlin/packit/config/WebConfig.kt index 1e0b48ad..c8977f15 100644 --- a/api/app/src/main/kotlin/packit/config/WebConfig.kt +++ b/api/app/src/main/kotlin/packit/config/WebConfig.kt @@ -1,10 +1,8 @@ package packit.config -import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer -import org.springframework.web.servlet.config.annotation.InterceptorRegistry import org.springframework.web.servlet.config.annotation.WebMvcConfigurer @Configuration diff --git a/api/app/src/main/kotlin/packit/controllers/PacketController.kt b/api/app/src/main/kotlin/packit/controllers/PacketController.kt index ef124fe9..5041d5d6 100644 --- a/api/app/src/main/kotlin/packit/controllers/PacketController.kt +++ b/api/app/src/main/kotlin/packit/controllers/PacketController.kt @@ -90,7 +90,7 @@ class PacketController( @PathVariable name: String, @RequestParam(required = true) hashes: List, @RequestParam(required = true) filenames: List, - ): ResponseEntity // Note: when using this option it is highly recommended to configure explicitly the TaskExecutor used in Spring MVC for executing asynchronous requests. Both the MVC Java config and the MVC namespaces provide options to configure asynchronous handling. If not using those, an application can set the taskExecutor property of RequestMappingHandlerAdapter. + ): ResponseEntity { val filesWithNames = hashes.mapIndexed { index, hash -> val fileResource = packetService.getFileByHash(hash, false, hash).first diff --git a/api/app/src/test/kotlin/packit/unit/controllers/PacketControllerTest.kt b/api/app/src/test/kotlin/packit/unit/controllers/PacketControllerTest.kt index 0fb17c15..4189ab75 100644 --- a/api/app/src/test/kotlin/packit/unit/controllers/PacketControllerTest.kt +++ b/api/app/src/test/kotlin/packit/unit/controllers/PacketControllerTest.kt @@ -15,6 +15,7 @@ import packit.model.* import packit.model.dto.PacketGroupSummary import packit.service.PacketGroupService import packit.service.PacketService +import packit.service.ZipService import java.time.Instant import kotlin.test.assertEquals @@ -90,11 +91,12 @@ class PacketControllerTest private val packetGroupService = mock { on { getPacketGroupSummaries(PageablePayload(0, 10), "") } doReturn mockPacketGroupsSummary } + private val zipService = mock() @Test fun `get pageable packets`() { - val sut = PacketController(packetService, packetGroupService) + val sut = PacketController(packetService, packetGroupService, zipService) val result = sut.pageableIndex(0, 10, "", "") assertEquals(result.statusCode, HttpStatus.OK) assertEquals(result.body, mockPageablePackets.map { it.toDto() }) @@ -105,7 +107,7 @@ class PacketControllerTest @Test fun `get packets by packet group name`() { - val sut = PacketController(packetService, packetGroupService) + val sut = PacketController(packetService, packetGroupService, zipService) val result = sut.getPacketsByName("pg1") @@ -117,7 +119,7 @@ class PacketControllerTest @Test fun `get packet groups summary`() { - val sut = PacketController(packetService, packetGroupService) + val sut = PacketController(packetService, packetGroupService, zipService) val result = sut.getPacketGroupSummaries(0, 10, "") assertEquals(result.statusCode, HttpStatus.OK) assertEquals(result.body, mockPacketGroupsSummary) @@ -127,7 +129,7 @@ class PacketControllerTest @Test fun `get packet metadata by id`() { - val sut = PacketController(packetService, packetGroupService) + val sut = PacketController(packetService, packetGroupService, zipService) val result = sut.findPacketMetadata("1") val responseBody = result.body assertEquals(result.statusCode, HttpStatus.OK) @@ -137,7 +139,7 @@ class PacketControllerTest @Test fun `get packet file by id`() { - val sut = PacketController(packetService, packetGroupService) + val sut = PacketController(packetService, packetGroupService, zipService) val result = sut.findFile("123", "sha123", false, "test.html") val responseBody = result.body diff --git a/app/src/app/components/contents/downloads/orderly/OtherFiles.tsx b/app/src/app/components/contents/downloads/orderly/OtherFiles.tsx index a93e4a2c..0660db90 100644 --- a/app/src/app/components/contents/downloads/orderly/OtherFiles.tsx +++ b/app/src/app/components/contents/downloads/orderly/OtherFiles.tsx @@ -25,20 +25,20 @@ export const OtherFiles = ({ inputFiles }: OtherFilesProps) => { - -
        - {inputsWithFileMetadata.map((input, index) => { - return ( - input.file && ( -
      • - -
        • - ) - ); - })} -
      -       -       + +
        + {inputsWithFileMetadata.map((input, index) => { + return ( + input.file && ( +
      • + +
        • + ) + ); + })} +
      +       +
      ); }; From 49c117a5736fdcce5179198e5dbbc93e8fe0548d Mon Sep 17 00:00:00 2001 From: David Mears Date: Wed, 29 Jan 2025 15:08:34 +0000 Subject: [PATCH 09/12] Fight warning 'calling non-final function in constructor'; fix typo --- api/app/src/main/kotlin/packit/AppConfig.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/app/src/main/kotlin/packit/AppConfig.kt b/api/app/src/main/kotlin/packit/AppConfig.kt index 5cb4a858..08cd13a1 100644 --- a/api/app/src/main/kotlin/packit/AppConfig.kt +++ b/api/app/src/main/kotlin/packit/AppConfig.kt @@ -8,14 +8,14 @@ import org.springframework.security.crypto.password.PasswordEncoder import org.springframework.stereotype.Component @Component -class AppConfig(private val enviroment: Environment) +class AppConfig(private val environment: Environment) { - internal fun requiredEnvValue(key: String): String + internal final fun requiredEnvValue(key: String): String { - return enviroment[key] ?: throw IllegalArgumentException("$key not set $enviroment") + return environment[key] ?: throw IllegalArgumentException("$key not set $environment") } - internal fun splitList(value: String): List + internal final fun splitList(value: String): List { if (value.isBlank()) { return listOf() From 7364c3e1046d68dbc27cd3900115cfcda8e7e5ba Mon Sep 17 00:00:00 2001 From: David Mears Date: Thu, 30 Jan 2025 12:37:23 +0000 Subject: [PATCH 10/12] Set some default config values for thread pool that handles async requests --- api/app/src/main/kotlin/packit/AppConfig.kt | 3 + .../packit/config/TaskExecutorProperties.kt | 56 +++++++++++++++++++ .../main/kotlin/packit/config/WebConfig.kt | 16 +++--- .../src/main/resources/application.properties | 6 ++ 4 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 api/app/src/main/kotlin/packit/config/TaskExecutorProperties.kt diff --git a/api/app/src/main/kotlin/packit/AppConfig.kt b/api/app/src/main/kotlin/packit/AppConfig.kt index 08cd13a1..971ba07b 100644 --- a/api/app/src/main/kotlin/packit/AppConfig.kt +++ b/api/app/src/main/kotlin/packit/AppConfig.kt @@ -1,13 +1,16 @@ package packit +import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.core.env.Environment import org.springframework.core.env.get import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder import org.springframework.security.crypto.password.PasswordEncoder import org.springframework.stereotype.Component +import packit.config.TaskExecutorProperties @Component +@EnableConfigurationProperties(TaskExecutorProperties::class) class AppConfig(private val environment: Environment) { internal final fun requiredEnvValue(key: String): String diff --git a/api/app/src/main/kotlin/packit/config/TaskExecutorProperties.kt b/api/app/src/main/kotlin/packit/config/TaskExecutorProperties.kt new file mode 100644 index 00000000..1c618e1a --- /dev/null +++ b/api/app/src/main/kotlin/packit/config/TaskExecutorProperties.kt @@ -0,0 +1,56 @@ +package packit.config + +import org.springframework.boot.context.properties.ConfigurationProperties +import org.springframework.context.annotation.Configuration + +/** +* Explicitly configures the thread pool for async requests, since explicit config is 'highly recommended' [1] for +* such requests. +* At time of writing, the only async request is thought to be /packets/{name}/{id}/zip, since it uses a +* StreamingResponseBody type [1], [2]; this motivated the introduction of this explicit configuration. +* +* The values here (and in application.properties) are defaults, which conservatively assume deployment on a lower-spec +* machine, and aim to minimize the risk of overloading the machine, rather than to optimize response time. +* If you know the properties of your hosting machine and context (how many threads can it handle concurrently? how +* significant are the bottlenecks of disk-reading I/O and network-writing I/O? how many users do you need to handle at +* once?), you can adjust these values to better fit your needs by setting the environment variables read in by +* application.properties. +* +* If the machine can theoretically handle a large number of threads, maximizing this won't necessarily improve +* throughput, since file-streaming is an I/O-bound operation, which means that threads will spend most of their time +* waiting for their turn to use the I/O (taking up about 1MB of RAM each, in the case of a 64-bit JVM), rather than +* doing any useful work with the CPU. +* +* References/docs: +* 1) https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/web/servlet/mvc/method/annotation/StreamingResponseBody.html +* 2) https://docs.spring.io/spring-framework/docs/4.3.12.RELEASE/spring-framework-reference/html/mvc.html#mvc-ann-async-output-stream +* 3) https://docs.spring.io/spring-framework/docs/4.3.12.RELEASE/spring-framework-reference/html/mvc.html#mvc-ann-async-configuration-spring-mvc +* 4) https://www.baeldung.com/thread-pool-java-and-guava +* 5) https://docs.spring.io/spring-framework/reference/integration/scheduling.html#scheduling-task-scheduler +* 6) https://codingtim.github.io/spring-threadpooltaskexecutor/ +*/ +@Configuration +@ConfigurationProperties(prefix = "packit.task-executor") +class TaskExecutorProperties { + // A number of core threads, which are always maintained for re-use. + val corePoolSize: Int = 4 + + // If the queue is full, and the core threads are all busy, new threads are created up to this limit, to handle + // bursts of load. + var maxPoolSize: Int = 10 + + // The queue fills up first, before any non-core thread is spawned. + var queueCapacity: Int = 100 + + // Number of seconds after which to clean up idle, non-core threads during periods of low load. + var keepAliveSeconds: Int = 60 + + // Under the 'caller runs policy', if the queue is full, and the core threads are all busy, the caller thread will + // run the task itself instead. This "provides a simple way to throttle the incoming load": see + // a) https://docs.spring.io/spring-framework/reference/integration/scheduling.html#scheduling-task-namespace-executor + // b) https://www.baeldung.com/java-rejectedexecutionhandler + // This throttling results from occupying the web container thread, which slows the rate it can accept new requests. + // If this property is set to false, the default policy is to abort. This may lead to clients re-trying, which could + // create even more load on the server. + var useCallerRunsPolicyToHandleRejectedExecution: Boolean = true +} \ No newline at end of file diff --git a/api/app/src/main/kotlin/packit/config/WebConfig.kt b/api/app/src/main/kotlin/packit/config/WebConfig.kt index c8977f15..1fdf093f 100644 --- a/api/app/src/main/kotlin/packit/config/WebConfig.kt +++ b/api/app/src/main/kotlin/packit/config/WebConfig.kt @@ -4,19 +4,21 @@ import org.springframework.context.annotation.Configuration import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer import org.springframework.web.servlet.config.annotation.WebMvcConfigurer +import java.util.concurrent.ThreadPoolExecutor @Configuration -class WebConfig : WebMvcConfigurer { - // Configure the thread pool for async requests. Explicit config is 'highly recommended'. See: - // https://docs.spring.io/spring-framework/docs/4.3.12.RELEASE/spring-framework-reference/html/mvc.html#mvc-ann-async-configuration-spring-mvc +class WebConfig(private val taskExecutorProperties: TaskExecutorProperties) : WebMvcConfigurer { override fun configureAsyncSupport(configurer: AsyncSupportConfigurer) { val executor = ThreadPoolTaskExecutor() - executor.corePoolSize = 10 - executor.maxPoolSize = 50 - executor.queueCapacity = 100 + executor.corePoolSize = taskExecutorProperties.corePoolSize + executor.maxPoolSize = taskExecutorProperties.maxPoolSize + executor.queueCapacity = taskExecutorProperties.queueCapacity executor.setThreadNamePrefix("Async-") + if (taskExecutorProperties.useCallerRunsPolicyToHandleRejectedExecution) { + executor.setRejectedExecutionHandler(ThreadPoolExecutor.CallerRunsPolicy()) + } executor.initialize() configurer.setTaskExecutor(executor) - configurer.setDefaultTimeout(60000) + configurer.setDefaultTimeout((taskExecutorProperties.keepAliveSeconds * 1000).toLong()) } } \ No newline at end of file diff --git a/api/app/src/main/resources/application.properties b/api/app/src/main/resources/application.properties index 13b1cc25..30a727e8 100644 --- a/api/app/src/main/resources/application.properties +++ b/api/app/src/main/resources/application.properties @@ -19,6 +19,12 @@ spring.mvc.throw-exception-if-no-handler-found=true spring.web.resources.add-mappings=false server.error.include-stacktrace=never # CUSTOM PACKIT CONFIG +#TaskExecutor +packit.task-executor.core-pool-size=${PACKIT_TASK_EXECUTOR_CORE_POOL_SIZE:4} +packit.task-executor.max-pool-size=${PACKIT_TASK_EXECUTOR_MAX_POOL_SIZE:10} +packit.task-executor.queue-capacity=${PACKIT_TASK_EXECUTOR_QUEUE_CAPACITY:100} +packit.task-executor.keep-alive-seconds=${PACKIT_TASK_EXECUTOR_KEEP_ALIVE_SECONDS:60} +packit.task-executor.use-caller-runs-policy-to-handle-rejected-execution=${PACKIT_TASK_EXECUTOR_USE_CALLER_RUNS_POLICY_TO_HANDLE_REJECTED_EXECUTION:true} #db outpack.server.url=${PACKIT_OUTPACK_SERVER_URL:http://localhost:8000} orderly.runner.url=${PACKIT_ORDERLY_RUNNER_URL:http://localhost:8001} From 7aab66bc41cf30bcd0e09346eb091b1ca91eb228 Mon Sep 17 00:00:00 2001 From: David Mears Date: Thu, 30 Jan 2025 16:31:56 +0000 Subject: [PATCH 11/12] Switch to using spring boot knobs to configure task execution --- api/app/src/main/kotlin/packit/AppConfig.kt | 3 +- .../packit/config/TaskExecutorProperties.kt | 56 ------------------- .../main/kotlin/packit/config/WebConfig.kt | 24 -------- .../src/main/resources/application.properties | 11 ++-- 4 files changed, 6 insertions(+), 88 deletions(-) delete mode 100644 api/app/src/main/kotlin/packit/config/TaskExecutorProperties.kt delete mode 100644 api/app/src/main/kotlin/packit/config/WebConfig.kt diff --git a/api/app/src/main/kotlin/packit/AppConfig.kt b/api/app/src/main/kotlin/packit/AppConfig.kt index 971ba07b..f0bfeaf3 100644 --- a/api/app/src/main/kotlin/packit/AppConfig.kt +++ b/api/app/src/main/kotlin/packit/AppConfig.kt @@ -7,10 +7,9 @@ import org.springframework.core.env.get import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder import org.springframework.security.crypto.password.PasswordEncoder import org.springframework.stereotype.Component -import packit.config.TaskExecutorProperties @Component -@EnableConfigurationProperties(TaskExecutorProperties::class) +@EnableConfigurationProperties() class AppConfig(private val environment: Environment) { internal final fun requiredEnvValue(key: String): String diff --git a/api/app/src/main/kotlin/packit/config/TaskExecutorProperties.kt b/api/app/src/main/kotlin/packit/config/TaskExecutorProperties.kt deleted file mode 100644 index 1c618e1a..00000000 --- a/api/app/src/main/kotlin/packit/config/TaskExecutorProperties.kt +++ /dev/null @@ -1,56 +0,0 @@ -package packit.config - -import org.springframework.boot.context.properties.ConfigurationProperties -import org.springframework.context.annotation.Configuration - -/** -* Explicitly configures the thread pool for async requests, since explicit config is 'highly recommended' [1] for -* such requests. -* At time of writing, the only async request is thought to be /packets/{name}/{id}/zip, since it uses a -* StreamingResponseBody type [1], [2]; this motivated the introduction of this explicit configuration. -* -* The values here (and in application.properties) are defaults, which conservatively assume deployment on a lower-spec -* machine, and aim to minimize the risk of overloading the machine, rather than to optimize response time. -* If you know the properties of your hosting machine and context (how many threads can it handle concurrently? how -* significant are the bottlenecks of disk-reading I/O and network-writing I/O? how many users do you need to handle at -* once?), you can adjust these values to better fit your needs by setting the environment variables read in by -* application.properties. -* -* If the machine can theoretically handle a large number of threads, maximizing this won't necessarily improve -* throughput, since file-streaming is an I/O-bound operation, which means that threads will spend most of their time -* waiting for their turn to use the I/O (taking up about 1MB of RAM each, in the case of a 64-bit JVM), rather than -* doing any useful work with the CPU. -* -* References/docs: -* 1) https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/web/servlet/mvc/method/annotation/StreamingResponseBody.html -* 2) https://docs.spring.io/spring-framework/docs/4.3.12.RELEASE/spring-framework-reference/html/mvc.html#mvc-ann-async-output-stream -* 3) https://docs.spring.io/spring-framework/docs/4.3.12.RELEASE/spring-framework-reference/html/mvc.html#mvc-ann-async-configuration-spring-mvc -* 4) https://www.baeldung.com/thread-pool-java-and-guava -* 5) https://docs.spring.io/spring-framework/reference/integration/scheduling.html#scheduling-task-scheduler -* 6) https://codingtim.github.io/spring-threadpooltaskexecutor/ -*/ -@Configuration -@ConfigurationProperties(prefix = "packit.task-executor") -class TaskExecutorProperties { - // A number of core threads, which are always maintained for re-use. - val corePoolSize: Int = 4 - - // If the queue is full, and the core threads are all busy, new threads are created up to this limit, to handle - // bursts of load. - var maxPoolSize: Int = 10 - - // The queue fills up first, before any non-core thread is spawned. - var queueCapacity: Int = 100 - - // Number of seconds after which to clean up idle, non-core threads during periods of low load. - var keepAliveSeconds: Int = 60 - - // Under the 'caller runs policy', if the queue is full, and the core threads are all busy, the caller thread will - // run the task itself instead. This "provides a simple way to throttle the incoming load": see - // a) https://docs.spring.io/spring-framework/reference/integration/scheduling.html#scheduling-task-namespace-executor - // b) https://www.baeldung.com/java-rejectedexecutionhandler - // This throttling results from occupying the web container thread, which slows the rate it can accept new requests. - // If this property is set to false, the default policy is to abort. This may lead to clients re-trying, which could - // create even more load on the server. - var useCallerRunsPolicyToHandleRejectedExecution: Boolean = true -} \ No newline at end of file diff --git a/api/app/src/main/kotlin/packit/config/WebConfig.kt b/api/app/src/main/kotlin/packit/config/WebConfig.kt deleted file mode 100644 index 1fdf093f..00000000 --- a/api/app/src/main/kotlin/packit/config/WebConfig.kt +++ /dev/null @@ -1,24 +0,0 @@ -package packit.config - -import org.springframework.context.annotation.Configuration -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor -import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer -import java.util.concurrent.ThreadPoolExecutor - -@Configuration -class WebConfig(private val taskExecutorProperties: TaskExecutorProperties) : WebMvcConfigurer { - override fun configureAsyncSupport(configurer: AsyncSupportConfigurer) { - val executor = ThreadPoolTaskExecutor() - executor.corePoolSize = taskExecutorProperties.corePoolSize - executor.maxPoolSize = taskExecutorProperties.maxPoolSize - executor.queueCapacity = taskExecutorProperties.queueCapacity - executor.setThreadNamePrefix("Async-") - if (taskExecutorProperties.useCallerRunsPolicyToHandleRejectedExecution) { - executor.setRejectedExecutionHandler(ThreadPoolExecutor.CallerRunsPolicy()) - } - executor.initialize() - configurer.setTaskExecutor(executor) - configurer.setDefaultTimeout((taskExecutorProperties.keepAliveSeconds * 1000).toLong()) - } -} \ No newline at end of file diff --git a/api/app/src/main/resources/application.properties b/api/app/src/main/resources/application.properties index 30a727e8..e1b23e24 100644 --- a/api/app/src/main/resources/application.properties +++ b/api/app/src/main/resources/application.properties @@ -18,13 +18,12 @@ management.endpoints.web.exposure.include=health,prometheus spring.mvc.throw-exception-if-no-handler-found=true spring.web.resources.add-mappings=false server.error.include-stacktrace=never +#task execution +spring.task.execution.pool.core-size=${PACKIT_TASK_EXECUTOR_CORE_POOL_SIZE:4} +spring.task.execution.pool.max-size=${PACKIT_TASK_EXECUTOR_MAX_POOL_SIZE:10} +spring.task.execution.pool.queue-capacity=${PACKIT_TASK_EXECUTOR_QUEUE_CAPACITY:100} +spring.task.execution.pool.keep-alive=${PACKIT_TASK_EXECUTOR_KEEP_ALIVE_SECONDS:60} # CUSTOM PACKIT CONFIG -#TaskExecutor -packit.task-executor.core-pool-size=${PACKIT_TASK_EXECUTOR_CORE_POOL_SIZE:4} -packit.task-executor.max-pool-size=${PACKIT_TASK_EXECUTOR_MAX_POOL_SIZE:10} -packit.task-executor.queue-capacity=${PACKIT_TASK_EXECUTOR_QUEUE_CAPACITY:100} -packit.task-executor.keep-alive-seconds=${PACKIT_TASK_EXECUTOR_KEEP_ALIVE_SECONDS:60} -packit.task-executor.use-caller-runs-policy-to-handle-rejected-execution=${PACKIT_TASK_EXECUTOR_USE_CALLER_RUNS_POLICY_TO_HANDLE_REJECTED_EXECUTION:true} #db outpack.server.url=${PACKIT_OUTPACK_SERVER_URL:http://localhost:8000} orderly.runner.url=${PACKIT_ORDERLY_RUNNER_URL:http://localhost:8001} From 8cbb94ddf5fef88163f330a8158fe6c90c5fac22 Mon Sep 17 00:00:00 2001 From: David Mears Date: Fri, 31 Jan 2025 16:49:50 +0000 Subject: [PATCH 12/12] Fix missing 's' in application.properties, and rely on 'externalized config' for env vars --- api/app/src/main/resources/application.properties | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/app/src/main/resources/application.properties b/api/app/src/main/resources/application.properties index e1b23e24..d15be6d4 100644 --- a/api/app/src/main/resources/application.properties +++ b/api/app/src/main/resources/application.properties @@ -19,10 +19,10 @@ spring.mvc.throw-exception-if-no-handler-found=true spring.web.resources.add-mappings=false server.error.include-stacktrace=never #task execution -spring.task.execution.pool.core-size=${PACKIT_TASK_EXECUTOR_CORE_POOL_SIZE:4} -spring.task.execution.pool.max-size=${PACKIT_TASK_EXECUTOR_MAX_POOL_SIZE:10} -spring.task.execution.pool.queue-capacity=${PACKIT_TASK_EXECUTOR_QUEUE_CAPACITY:100} -spring.task.execution.pool.keep-alive=${PACKIT_TASK_EXECUTOR_KEEP_ALIVE_SECONDS:60} +spring.task.execution.pool.core-size=4 +spring.task.execution.pool.max-size=10 +spring.task.execution.pool.queue-capacity=100 +spring.task.execution.pool.keep-alive=60s # CUSTOM PACKIT CONFIG #db outpack.server.url=${PACKIT_OUTPACK_SERVER_URL:http://localhost:8000}