Merge 4f8b013 into 7dfbc7d

.env.example

@@ -39,6 +39,8 @@ FILE_API_KEY_DONCASTER=👻
 FILE_API_KEY_GLOUCESTER=👻
 FILE_API_KEY_TEWKESBURY=👻
 
+# Used to circumvent API rate limiting for development purposes (e.g. load testing)
+SKIP_RATE_LIMIT_SECRET=👻
 
 # Editor
 EDITOR_URL_EXT=http://localhost:3000

api.planx.uk/modules/auth/fixtures/microsoft-openid-configuration.json

@@ -0,0 +1,54 @@
+{
+  "token_endpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/token",
+  "token_endpoint_auth_methods_supported": [
+    "client_secret_post",
+    "private_key_jwt",
+    "client_secret_basic"
+  ],
+  "jwks_uri": "https://login.microsoftonline.com/common/discovery/v2.0/keys",
+  "response_modes_supported": ["query", "fragment", "form_post"],
+  "subject_types_supported": ["pairwise"],
+  "id_token_signing_alg_values_supported": ["RS256"],
+  "response_types_supported": [
+    "code",
+    "id_token",
+    "code id_token",
+    "id_token token"
+  ],
+  "scopes_supported": ["openid", "profile", "email", "offline_access"],
+  "issuer": "https://login.microsoftonline.com/common/v2.0",
+  "request_uri_parameter_supported": false,
+  "userinfo_endpoint": "https://graph.microsoft.com/oidc/userinfo",
+  "authorization_endpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
+  "device_authorization_endpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/devicecode",
+  "http_logout_supported": true,
+  "frontchannel_logout_supported": true,
+  "end_session_endpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/logout",
+  "claims_supported": [
+    "sub",
+    "iss",
+    "cloud_instance_name",
+    "cloud_instance_host_name",
+    "cloud_graph_host_name",
+    "msgraph_host",
+    "aud",
+    "exp",
+    "iat",
+    "auth_time",
+    "acr",
+    "nonce",
+    "preferred_username",
+    "name",
+    "tid",
+    "ver",
+    "at_hash",
+    "c_hash",
+    "email"
+  ],
+  "kerberos_endpoint": "https://login.microsoftonline.com/common/kerberos",
+  "tenant_region_scope": null,
+  "cloud_instance_name": "microsoftonline.com",
+  "cloud_graph_host_name": "graph.windows.net",
+  "msgraph_host": "graph.microsoft.com",
+  "rbac_url": "https://pas.windows.net"
+}

api.planx.uk/modules/auth/passport.ts

@@ -1,4 +1,9 @@
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+
 import { Issuer } from "openid-client";
+import type { IssuerMetadata } from "openid-client";
 import type { Authenticator } from "passport";
 import passport from "passport";
 
@@ -14,7 +19,29 @@ export default async (): Promise<Authenticator> => {
   const customPassport = new passport.Passport();
 
   // instantiate Microsoft OIDC client, and use it to build the related strategy
-  const microsoftIssuer = await Issuer.discover(MICROSOFT_OPENID_CONFIG_URL);
+  // we also keep said config as a fixture to enable offline local development
+  let microsoftIssuer;
+  if (
+    process.env.APP_ENVIRONMENT == "development" &&
+    process.env.DEVELOP_OFFLINE
+  ) {
+    console.info(
+      "Working offline: using saved Microsoft OIDC configuration in auth/fixtures",
+    );
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = path.dirname(__filename);
+    const fixturePath = path.resolve(
+      __dirname,
+      "fixtures",
+      "microsoft-openid-configuration.json",
+    );
+    const microsoftIssuerConfig: IssuerMetadata = JSON.parse(
+      fs.readFileSync(fixturePath, "utf-8"),
+    );
+    microsoftIssuer = new Issuer(microsoftIssuerConfig);
+  } else {
+    microsoftIssuer = await Issuer.discover(MICROSOFT_OPENID_CONFIG_URL);
+  }
   console.debug("Discovered issuer %s", microsoftIssuer.issuer);
   const microsoftOidcClient = new microsoftIssuer.Client(
     getMicrosoftClientConfig(),

api.planx.uk/modules/file/controller.ts

@@ -1,4 +1,5 @@
 import assert from "assert";
+import { deleteFilesByKey } from "./service/deleteFile.js";
 import { uploadPrivateFile, uploadPublicFile } from "./service/uploadFile.js";
 import { buildFilePath } from "./service/utils.js";
 import { getFileFromS3 } from "./service/getFile.js";
@@ -71,15 +72,15 @@ export const publicUploadController: UploadController = async (
   }
 };
 
-export const downloadFileSchema = z.object({
+export const hostedFileSchema = z.object({
   params: z.object({
     fileKey: z.string(),
     fileName: z.string(),
   }),
 });
 
 export type DownloadController = ValidatedRequestHandler<
-  typeof downloadFileSchema,
+  typeof hostedFileSchema,
   Buffer | undefined
 >;
 
@@ -124,3 +125,30 @@ export const privateDownloadController: DownloadController = async (
     );
   }
 };
+
+export type DeleteController = ValidatedRequestHandler<
+  typeof hostedFileSchema,
+  Record<string, never>
+>;
+
+export const publicDeleteController: DeleteController = async (
+  _req,
+  res,
+  next,
+) => {
+  const { fileKey, fileName } = res.locals.parsedReq.params;
+  const filePath = buildFilePath(fileKey, fileName);
+
+  try {
+    const { isPrivate } = await getFileFromS3(filePath);
+    if (isPrivate) throw Error("Bad request");
+
+    // once we've established that the file is public, we can delete it
+    await deleteFilesByKey([filePath]);
+    res.status(204).send();
+  } catch (error) {
+    return next(
+      new ServerError({ message: `Failed to delete public file: ${error}` }),
+    );
+  }
+};

api.planx.uk/modules/file/docs.yaml

@@ -4,7 +4,7 @@ info:
   version: 0.1.0
 tags:
   - name: file
-    description: Endpoints for uploading and downloading files
+    description: Endpoints for uploading, downloading and deleting files
 components:
   parameters:
     fileKey:
@@ -84,6 +84,18 @@ paths:
           $ref: "#/components/responses/DownloadFile"
         "500":
           $ref: "#/components/responses/ErrorMessage"
+    delete:
+      tags: ["file"]
+      parameters:
+        - $ref: "#/components/parameters/fileKey"
+        - $ref: "#/components/parameters/fileName"
+      security:
+        - bearerAuth: []
+      responses:
+        "204":
+          $ref: "#/components/responses/UploadFile"
+        "500":
+          $ref: "#/components/responses/ErrorMessage"
   /file/private/{fileKey}/{fileName}:
     get:
       tags: ["file"]
@@ -93,7 +105,7 @@ paths:
       security:
         - fileAPIKeyAuth: []
       responses:
-        "200":
-          $ref: "#/components/responses/DownloadFile"
+        "204":
+          description: Successful deletion
         "500":
           $ref: "#/components/responses/ErrorMessage"

api.planx.uk/modules/file/routes.ts

@@ -1,15 +1,16 @@
 import { Router } from "express";
 
-import multer from "multer";
 import {
-  useNoCache,
   useFilePermission,
+  useNoCache,
+  usePlatformAdminAuth,
   useTeamEditorAuth,
 } from "../auth/middleware.js";
 import {
-  downloadFileSchema,
+  hostedFileSchema,
   privateDownloadController,
   privateUploadController,
+  publicDeleteController,
   publicDownloadController,
   publicUploadController,
   uploadFileSchema,
@@ -36,16 +37,23 @@ router.post(
 
 router.get(
   "/file/public/:fileKey/:fileName",
-  validate(downloadFileSchema),
+  validate(hostedFileSchema),
   publicDownloadController,
 );
 
 router.get(
   "/file/private/:fileKey/:fileName",
   useNoCache,
   useFilePermission,
-  validate(downloadFileSchema),
+  validate(hostedFileSchema),
   privateDownloadController,
 );
 
+router.delete(
+  "/file/public/:fileKey/:fileName",
+  usePlatformAdminAuth,
+  validate(hostedFileSchema),
+  publicDeleteController,
+);
+
 export default router;

api.planx.uk/modules/file/service/uploadFile.ts

@@ -1,5 +1,6 @@
 import {
   GetObjectCommand,
+  type S3,
   type PutObjectCommandInput,
 } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
@@ -19,7 +20,7 @@ export const uploadPublicFile = async (
   const { params, key, fileType } = generateFileParams(file, filename, filekey);
 
   await s3.putObject(params);
-  const fileUrl = await buildFileUrl(key, "public");
+  const fileUrl = await buildFileUrl(s3, key, "public");
 
   return {
     fileType,
@@ -41,7 +42,7 @@ export const uploadPrivateFile = async (
   };
 
   await s3.putObject(params);
-  const fileUrl = await buildFileUrl(key, "private");
+  const fileUrl = await buildFileUrl(s3, key, "private");
 
   return {
     fileType,
@@ -50,8 +51,11 @@ export const uploadPrivateFile = async (
 };
 
 // Construct an API URL for the uploaded file
-const buildFileUrl = async (key: string, path: "public" | "private") => {
-  const s3 = s3Factory();
+const buildFileUrl = async (
+  s3: S3,
+  key: string,
+  path: "public" | "private",
+) => {
   const s3Url = await getSignedUrl(
     s3,
     new GetObjectCommand({ Key: key, Bucket: process.env.AWS_S3_BUCKET }),
@@ -85,8 +89,8 @@ export function generateFileParams(
   };
 
   return {
-    fileType,
     params,
     key,
+    fileType,
   };
 }

api.planx.uk/modules/file/service/utils.ts

@@ -23,7 +23,7 @@ function useMinio() {
     // Points to Minio
     return {
       endpoint: `http://minio:${process.env.MINIO_PORT}`,
-      s3ForcePathStyle: true,
+      forcePathStyle: true,
       signatureVersion: "v4",
     };
   }

api.planx.uk/rateLimit.ts

@@ -7,6 +7,19 @@ const apiLimiter = rateLimit({
   max: 250,
   standardHeaders: true,
   legacyHeaders: false,
+  skip: (req: Request, _res: Response) => {
+    // add a mechanism for skipping rate limit when load testing (on local or staging only)
+    if (process.env.APP_ENVIRONMENT == "production") return false;
+    const rateLimitHeader = req.get("X-Skip-Rate-Limit-Secret");
+    const SKIP_RATE_LIMIT_SECRET = process.env?.SKIP_RATE_LIMIT_SECRET;
+    if (
+      rateLimitHeader &&
+      SKIP_RATE_LIMIT_SECRET &&
+      rateLimitHeader === SKIP_RATE_LIMIT_SECRET
+    )
+      return true;
+    return false;
+  },
 });
 
 const HASURA_ONLY_SEND_EMAIL_TEMPLATES = ["reminder", "expiry"];

docker-compose.yml

@@ -156,6 +156,7 @@ services:
       ORDNANCE_SURVEY_API_KEY: ${ORDNANCE_SURVEY_API_KEY}
       PORT: ${API_PORT}
       SESSION_SECRET: ${SESSION_SECRET}
+      SKIP_RATE_LIMIT_SECRET: ${SKIP_RATE_LIMIT_SECRET}
       SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
       UNIFORM_SUBMISSION_URL: ${UNIFORM_SUBMISSION_URL}
       UNIFORM_TOKEN_URL: ${UNIFORM_TOKEN_URL}

infrastructure/application/Pulumi.staging.yaml

@@ -35,6 +35,8 @@ config:
     secure: AAABAK8LsYKKNgIS4fepW5Sh6+WKxNopxsos51eBttT7O8E8K0HYOswgrIWuYJ0R1eJHDLKRHqQ=
   application:file-api-key-tewkesbury:
     secure: AAABALlStpxyNG5SRQFVYJMGmCyteUkoU9XBTBJn2kcf6APdqO1JwxU4jiU9Qo6a6aZQXK60an7xbkuD2hla/UvjR7Wu7cXY
+  application:skip-rate-limit-secret:
+    secure: AAABANwjBmGN73bwF6nqQP5i8tKMe3mi0NMBdae4uaLasG2VfPvC1D4mk7QWIFqVnzsD5jCN20Eqos/r2B+EAo2rAws3JeD/
   application:google-client-id: 987324067365-vpsk3kgeq5n32ihjn760ihf8l7m5rhh8.apps.googleusercontent.com
   application:google-client-secret:
     secure: AAABAGQuqQDU4S+vR+cQaFoa6xAeWU9clVaNonQ/dq0R8Dke+o0y7ALOmYMy4fOX4Pa6HiZl85npU/cbwy8HdMYaiA==

infrastructure/application/index.ts

@@ -392,6 +392,10 @@ export = async () => {
             name: "FILE_API_KEY_TEWKESBURY",
             value: config.requireSecret("file-api-key-tewkesbury"),
           },
+          {
+            name: "SKIP_RATE_LIMIT_SECRET",
+            value: config.requireSecret("skip-rate-limit-secret"),
+          },
           {
             name: "GOOGLE_CLIENT_ID",
             value: config.require("google-client-id"),

infrastructure/performance/README.md

@@ -1,18 +1,16 @@
-# Performance
-
-## Load testing with Locust
+# Load testing with Locust
 
 This directory contains Python scripts for load testing using [Locust](https://locust.io/) ([docs](https://docs.locust.io/en/stable/)).
 
-### Setup
+## Setup
 
 We use `uv` to manage dependencies for this project. If you aren't already familiar with this Python project manager, [get set up](https://docs.astral.sh/uv/).
 
 Then:
 - run `uv sync` (`pyproject.toml` and `uv.lock` together completely determine the setup)
 - run `source .venv/bin/activate` to [activate the virtual environment](https://docs.astral.sh/uv/pip/environments/#using-a-virtual-environment)
 
-### Usage
+## Usage
 
 The `run_locust.sh` script is intended to encode some sensible assumptions and do some of the heavy lifting to make it very easy to run load tests.
 
@@ -26,13 +24,23 @@ As an example, the following command will simulate 500 users hitting PlanX stagi
 
 Then find the Locust GUI at `http://localhost:8089/`.
 
-### Development
+## Development
 
 The `OpenWorkloadBase` class in `base_workload.py` provides a base class which all the `test_*.py` scripts inherit from. Any new workload should follow the same pattern.
 
-Also note that this project using [ruff](https://docs.astral.sh/ruff/) for linting and formatting. So before pushing up changes (and with the venv activated), run the following:
+Also note that this project uses [ruff](https://docs.astral.sh/ruff/) for linting and formatting. So before pushing up changes (and with the venv activated), run the following:
 
 ```
 ruff check
 ruff format
 ```
+
+### Auth
+
+Some workloads may require authentication, e.g. `test_api.py`. To get this working, just log in to any environment, grab the JWT and export it in your shell as an `AUTH_TOKEN` environment variable. The same script will only be able to load test staging in a serious fashion if it is also supplied with `SKIP_RATE_LIMIT_SECRET`, which should be in your `.env` file at root of project.
+
+### Samples
+
+The API load testing script requires some files to work with, which are in `/samples`. Care should be taken to make sure anything added there is in the public domain.
+
+For example, I used [Unsplash](https://unsplash.com/s/photos/tree?license=free) to search for an image with a ['free' license](https://unsplash.com/license), and printed a page from the [WikiHouse](https://www.wikihouse.cc/) site as a PDF.