WIP: add API load testing script, enable easier offline development o…

…f API

api.planx.uk/modules/auth/fixtures/microsoft-openid-configuration.json

@@ -0,0 +1,54 @@
+{
+  "token_endpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/token",
+  "token_endpoint_auth_methods_supported": [
+    "client_secret_post",
+    "private_key_jwt",
+    "client_secret_basic"
+  ],
+  "jwks_uri": "https://login.microsoftonline.com/common/discovery/v2.0/keys",
+  "response_modes_supported": ["query", "fragment", "form_post"],
+  "subject_types_supported": ["pairwise"],
+  "id_token_signing_alg_values_supported": ["RS256"],
+  "response_types_supported": [
+    "code",
+    "id_token",
+    "code id_token",
+    "id_token token"
+  ],
+  "scopes_supported": ["openid", "profile", "email", "offline_access"],
+  "issuer": "https://login.microsoftonline.com/common/v2.0",
+  "request_uri_parameter_supported": false,
+  "userinfo_endpoint": "https://graph.microsoft.com/oidc/userinfo",
+  "authorization_endpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
+  "device_authorization_endpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/devicecode",
+  "http_logout_supported": true,
+  "frontchannel_logout_supported": true,
+  "end_session_endpoint": "https://login.microsoftonline.com/common/oauth2/v2.0/logout",
+  "claims_supported": [
+    "sub",
+    "iss",
+    "cloud_instance_name",
+    "cloud_instance_host_name",
+    "cloud_graph_host_name",
+    "msgraph_host",
+    "aud",
+    "exp",
+    "iat",
+    "auth_time",
+    "acr",
+    "nonce",
+    "preferred_username",
+    "name",
+    "tid",
+    "ver",
+    "at_hash",
+    "c_hash",
+    "email"
+  ],
+  "kerberos_endpoint": "https://login.microsoftonline.com/common/kerberos",
+  "tenant_region_scope": null,
+  "cloud_instance_name": "microsoftonline.com",
+  "cloud_graph_host_name": "graph.windows.net",
+  "msgraph_host": "graph.microsoft.com",
+  "rbac_url": "https://pas.windows.net"
+}

api.planx.uk/modules/auth/passport.ts

@@ -1,4 +1,9 @@
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+
 import { Issuer } from "openid-client";
+import type { IssuerMetadata } from "openid-client";
 import type { Authenticator } from "passport";
 import passport from "passport";
 
@@ -14,7 +19,29 @@ export default async (): Promise<Authenticator> => {
   const customPassport = new passport.Passport();
 
   // instantiate Microsoft OIDC client, and use it to build the related strategy
-  const microsoftIssuer = await Issuer.discover(MICROSOFT_OPENID_CONFIG_URL);
+  // we also keep said config as a fixture to enable offline local development
+  let microsoftIssuer;
+  if (
+    process.env.APP_ENVIRONMENT == "development" &&
+    process.env.DEVELOP_OFFLINE
+  ) {
+    console.info(
+      "Working offline: using saved Microsoft OIDC configuration in auth/fixtures",
+    );
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = path.dirname(__filename);
+    const fixturePath = path.resolve(
+      __dirname,
+      "fixtures",
+      "microsoft-openid-configuration.json",
+    );
+    const microsoftIssuerConfig: IssuerMetadata = JSON.parse(
+      fs.readFileSync(fixturePath, "utf-8"),
+    );
+    microsoftIssuer = new Issuer(microsoftIssuerConfig);
+  } else {
+    microsoftIssuer = await Issuer.discover(MICROSOFT_OPENID_CONFIG_URL);
+  }
   console.debug("Discovered issuer %s", microsoftIssuer.issuer);
   const microsoftOidcClient = new microsoftIssuer.Client(
     getMicrosoftClientConfig(),

infrastructure/performance/README.md

@@ -36,3 +36,9 @@ Also note that this project using [ruff](https://docs.astral.sh/ruff/) for linti
 ruff check
 ruff format
 ```
+
+### Samples
+
+The API load testing script requires some files to work with, which are in `/samples`. Care should be taken to make sure anything added there is in the public domain.
+
+For example, I used [Unsplash](https://unsplash.com/s/photos/tree?license=free) to search for an image with a ['free' license](https://unsplash.com/license), and printed a page from the [WikiHouse](https://www.wikihouse.cc/) site as a PDF.

infrastructure/performance/test_api.py

@@ -0,0 +1,54 @@
+from locust import (
+  constant_pacing,
+  task,
+)
+
+from base_workload import OpenWorkloadBase
+from utils import (
+  get_mime_type_from_filename,
+  get_random_file_from_dir,
+  get_target_host,
+)
+
+
+TASK_INVOCATION_RATE_SECONDS = 10
+HOST_BY_ENV = {
+  # "local": os.getenv("API_URL_EXT", "http://localhost:7002"),
+  "local": "http://localhost:8001",
+  "staging": "https://api.editor.planx.dev",
+}
+SAMPLE_FILE_DIRECTORY = "samples"
+MIME_TYPE_BY_FILE_EXT = {
+  "jpg": "image/jpeg",
+  "jpeg": "image/jpeg",
+  "png": "image/png",
+  "pdf": "application/pdf",
+}
+AUTH_JWT = "your-jwt-here"
+
+
+class APIWorkload(OpenWorkloadBase):
+  wait_time = constant_pacing(TASK_INVOCATION_RATE_SECONDS)
+  host = get_target_host(HOST_BY_ENV)
+
+  def on_start(self):
+    # need to auth in order to upload anything
+    pass
+
+  @task
+  def upload_public_file(self) -> None:
+    # it is essentially free to upload files to S3, and also free to delete them
+    # however it costs more to keep it there and to pull it down, so we don't load test that aspect
+    # we want to test a range of file types and sizes (although none should be larger than 30MB)
+    filename, file_bin = get_random_file_from_dir(SAMPLE_FILE_DIRECTORY)
+    with self.rest(
+      "POST",
+      "/file/public/upload",
+      cookies={"jwt": AUTH_JWT},
+      files={"file": (filename, file_bin, get_mime_type_from_filename(filename))},
+    ) as resp:
+      print(resp)
+
+  # @task
+  # def handle_submission(self) -> None:
+  #   pass

infrastructure/performance/utils.py

@@ -1,8 +1,15 @@
 import os
 from typing import Any
+import random
 
 
 VALID_TARGET_ENVIRONMENTS = ("local", "staging")
+MIME_TYPE_BY_FILE_EXT = {
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".png": "image/png",
+  ".pdf": "application/pdf",
+}
 
 
 def get_nested_key(dct: dict[Any, Any], *keys: str) -> Any:
@@ -19,3 +26,23 @@ def get_target_host(host_by_env: dict[str, str]) -> str:
   if env not in VALID_TARGET_ENVIRONMENTS:
     raise ValueError(f"Invalid environment submitted (accepts local/staging): {env}")
   return host_by_env[env]
+
+
+def get_random_file_from_dir(target_dir: str) -> [str, bytes]:
+  files = os.listdir(target_dir)
+  random_file = random.choice(files)
+  with open(os.path.join(target_dir, random_file), "rb") as f:
+    return random_file, f.read()
+
+
+def get_mime_type_from_filename(filename: str) -> str:
+  """
+  >>> get_mime_type_from_filename("photo.jpg")
+  'image/jpeg'
+  >>> get_mime_type_from_filename("document.pdf")
+  'application/pdf'
+  >>> get_mime_type_from_filename("unknown.svg")
+  'application/octet-stream'
+  """
+  _, file_ext = os.path.splitext(filename)
+  return MIME_TYPE_BY_FILE_EXT.get(file_ext, "application/octet-stream")