Fix issues with templates & makefile and update documentation

Author: mbklein

.tool-versions

@@ -1,4 +1,4 @@
 nodejs 20.15.0
 java corretto-19.0.1.10.1
-aws-sam-cli 1.107.0
+aws-sam-cli 1.135.0
 python 3.12.2

Makefile

@@ -5,27 +5,33 @@ ENV=dev
 SHELL := /bin/bash
 
 help:
-	echo "make build           | build the SAM project"
-	echo "make serve           | alias for serve-https"
-	echo "make clean           | remove all installed dependencies and build artifacts"
-	echo "make deps            | install all dependencies"
-	echo "make link            | create hard links to allow for hot reloading of a built project"
-	echo "make secrets         | symlink secrets files from ../tfvars"
-	echo "make start-with-step | run the SAM server locally with step function & download lambdas"
-	echo "make style           | run all style checks"
-	echo "make test            | run all tests"
-	echo "make cover           | run all tests with coverage"
-	echo "make env ENV=[env]   | activate env.\$$ENV.json file (default: dev)"
-	echo "make deps-node       | install node dependencies"
-	echo "make deps-python     | install python dependencies"
-	echo "make serve-http      | run the SAM server locally (HTTP on port 3000)"
-	echo "make serve-https     | run the SAM server locally (HTTPS on port 3002)"
-	echo "make style-node      | run node code style check"
-	echo "make style-python    | run python code style check"
-	echo "make test-node       | run node tests"
-	echo "make test-python     | run python tests"
-	echo "make cover-node      | run node tests with coverage"
-	echo "make cover-python    | run python tests with coverage"
+	echo "make build                 | build the SAM project"
+	echo "make serve                 | alias for serve-https"
+	echo "make clean                 | remove all installed dependencies and build artifacts"
+	echo "make deps                  | install all dependencies"
+	echo "make env.json 						 | create an env.json file for the current user's environment"
+	echo "make link                  | create hard links to allow for hot reloading of a built project"
+	echo "make secrets               | symlink secrets files from ../tfvars"
+	echo "make start-with-step       | run the SAM server locally with step function & download lambdas"
+	echo "make style                 | run all style checks"
+	echo "make test                  | run all tests"
+	echo "make cover                 | run all tests with coverage"
+	echo "make env ENV=[env]         | activate env.\$$ENV.json file (default: dev)"
+	echo "make deps-node             | install node dependencies"
+	echo "make deps-python           | install python dependencies"
+	echo "make samconfig.NAME.yaml"  | create a user samconfig file for the specified username"
+	echo "make build								 | build the SAM project for deploying"
+	echo "make deploy                | deploy the SAM project to AWS"
+	echo "make sync                  | sync the SAM project to AWS for quick development"
+	echo "make sync-code             | sync the SAM project to AWS (code changes only)"
+	echo "make serve-http            | run the SAM server locally (HTTP on port 3000)"
+	echo "make serve-https           | run the SAM server locally (HTTPS on port 3002)"
+	echo "make style-node            | run node code style check"
+	echo "make style-python          | run python code style check"
+	echo "make test-node             | run node tests"
+	echo "make test-python           | run python tests"
+	echo "make cover-node            | run node tests with coverage"
+	echo "make cover-python          | run python tests with coverage"
 
 .aws-sam/build.toml: ./template.yaml api/package-lock.json api/src/package-lock.json chat/dependencies/requirements.txt chat/src/requirements.txt
 	sed -Ei.orig 's/"dependencies"/"devDependencies"/' api/src/package.json
@@ -42,17 +48,18 @@ help:
 	done
 	mv api/src/package.json.orig api/src/package.json
 	mv api/src/package-lock.json.orig api/src/package-lock.json
-deps-node:
+deps-api:
 	cd api/src ;\
 	npm list >/dev/null 2>&1 ;\
 	src_deps=$$? ;\
 	cd .. ;\
 	npm list >/dev/null 2>&1 ;\
 	dev_deps=$$? ;\
 	test $$src_deps -eq 0 -a $$dev_deps -eq 0 || npm ci
-
+deps-av-download:
 	cd av-download/lambdas ;\
 	npm list >/dev/null 2>&1 || npm ci
+deps-node: deps-api deps-av-download
 cover-node: deps-node
 	cd api && npm run test:coverage
 style-node: deps-node
@@ -76,7 +83,7 @@ python-version:
 build:
 	bash -c "trap 'trap - SIGINT SIGTERM ERR; $(MAKE) reset; exit 1' SIGINT SIGTERM ERR; $(MAKE) .aws-sam/build.toml reset"
 validate:
-	cfn-lint template.yaml **/template.yaml --ignore-checks E3510
+	cfn-lint template.yaml **/template.yaml --ignore-checks E3510 W1028 W8001
 serve-http: deps-node env.json
 	@printf '\033[0;31mWARNING: Serving only the local HTTP API. The chat websocket API is not available in local mode.\033[0m\n'
 	rm -rf .aws-sam
@@ -87,12 +94,20 @@ serve: serve-https
 start-with-step: deps-node env.json
 	sam local start-lambda -t av-download/template.yaml --host 0.0.0.0 --port 3005 --env-vars $$PWD/env.json --log-file lambda.log & \
 	echo $$! > .sam-pids ;\
+	sg open all 3005 ;\
 	sam local start-api -t api/template.yaml --env-vars $$PWD/env.json --host 0.0.0.0 --port 3002 --log-file dc-api.log \
 		--ssl-cert-file $$HOME/.dev_cert/dev.rdc.cert.pem --ssl-key-file $$HOME/.dev_cert/dev.rdc.key.pem & \
 	echo $$! >> .sam-pids ;\
 	docker run --rm -p 8083:8083 -e LAMBDA_ENDPOINT=http://172.17.0.1:3005/ amazon/aws-stepfunctions-local ;\
+	echo -n "Shutting down..." ;\
+	sg close all 3005 ;\
 	kill $$(cat .sam-pids) ;\
-	rm -f .sam-pids
+	rm -f .sam-pids ;\
+	echo ""
+state-machine:
+	export TEMPLATE_DIR=$$(mktemp -d); \
+	yq -o=json '.Resources.avDownloadStateMachine.Properties.Definition' av-download/template.yaml > $$TEMPLATE_DIR/av_download.json; \
+	aws stepfunctions create-state-machine --endpoint http://localhost:8083 --definition file://$$TEMPLATE_DIR/av_download.json --name "hlsStitcherStepFunction" --role-arn arn:aws:iam::012345678901:role/DummyRole
 deps: deps-node deps-python
 style: style-node style-python
 test: test-node test-python
@@ -104,10 +119,12 @@ samconfig.%.yaml:
 deploy: build samconfig.$(DEV_PREFIX).yaml
 	sam deploy --config-file samconfig.$(DEV_PREFIX).yaml --stack-name dc-api-$(DEV_PREFIX)
 sync: samconfig.$(DEV_PREFIX).yaml
-	sam sync --config-file samconfig.$(DEV_PREFIX).yaml --stack-name dc-api-$(DEV_PREFIX) --watch
+	sam sync --config-file samconfig.$(DEV_PREFIX).yaml --stack-name dc-api-$(DEV_PREFIX) --watch $(ARGS)
+sync-code: ARGS=--code
+sync-code: sync
 secrets:
 	ln -s ../tfvars/dc-api/*.yaml .
 clean:
-	rm -rf .aws-sam api/node_modules api/src/node_modules python/**/__pycache__ python/.coverage python/.ruff_cache
+	rm -rf .aws-sam api/.aws-sam chat/.aws-sam av-download/.aws-sam api/node_modules api/src/node_modules chat/**/__pycache__ chat/.coverage chat/.ruff_cache
 reset:
 	for f in $$(find . -maxdepth 2 -name '*.orig'); do mv $$f $${f%%.orig}; done

README.md

@@ -12,7 +12,7 @@ The `env.json` file contains environment variable values for the lambda function
 make env.json
 ```
 
-If the file already exists, it will not be overwritten, so make sure to `rm` it if you would like to re-create it.
+If the file already exists, it will not be overwritten unless you include `-B` in the make command.
 
 ## Running the API locally
 
@@ -79,21 +79,11 @@ Access the app in a browser at: https://USER_PREFIX.dev.rdc.library.northwestern
 # From the repo root
 cd dc-api-v2
 
-#  Make sure you've done an `npm install` recently to update any packages in the `lambdas` directory
-npm install
-
-# Open port 3005 (if needed)
-sg open all 3005
-
-# Login as the staging-admin user
-export AWS_PROFILE=staging-admin
-aws sso login
-
 # Start the API + step function and associated lambdas
 make start-with-step
 
 # Open a second terminal and create the state machine
-aws stepfunctions create-state-machine --endpoint http://localhost:8083 --definition file://state_machines/av_download.json --name "hlsStitcherStepFunction" --role-arn arn:aws:iam::012345678901:role/DummyRole
+make state-machine
 ```
 
 ## Deploying a development branch
@@ -107,6 +97,9 @@ There are two ways to deploy a development branch: `make deploy` and `make sync`
 
 Either way, the resulting stack will be accessible at `https://dcapi-USER_PREFIX.rdc-staging.library.northwestern.edu`.
 
+An existing `sync` stack can be reused by running `make sync` again, or by running `make sync-code` to only
+sync code changes (no infrastructure/template changes).
+
 ### `samconfig.*.yaml`
 
 Both methods involve a `samconfig.USER_PREFIX.yaml` file. This file, with default values, can be created by
@@ -116,19 +109,20 @@ running (for example):
 make samconfig.mbk.yaml
 ```
 
-This will create a configuration to deploy the API, AV Download, and Chat stacks. To deploy a different combination of
-features, specify them using the `WITH` option:
+This will create a configuration to stand up the default stacks in both `deploy` mode (API, AV Download, and Chat) and
+`sync` mode (Chat only). To deploy a different combination of features, specify them using the `WITH` option:
 
 ```shell
 make samconfig.mbk.yaml WITH=API,DOCS
 ```
 
-Available features are: `API`, `AV_DOWNLOAD`, `CHAT`, and `DOCS`.
+Available features are: `API`, `AV_DOWNLOAD`, `CHAT`, and `DOCS`. 
+
+⚠️ Be **very** careful including the API in `sync` mode as every change within `/api` will take a long time to deploy.
 
-As with the `env.json` file, `make` will not overwrite the file if it already exists. You will need to `rm` it
-manually if you'd like to create a new one.
+As with the `env.json` file, `make` will not overwrite an existing file unless you include `-B`.
 
-### Tearing down a development branch
+### Tearing down a development stack
 
 ```shell
 sam delete --stack-name dc-api-USER_PREFIX

api/src/environment.js

@@ -1,37 +1,10 @@
 const fs = require("fs");
 const jwt = require("jsonwebtoken");
 const path = require("path");
-const {
-  GetSecretValueCommand,
-  SecretsManagerClient,
-} = require("@aws-sdk/client-secrets-manager");
 const PackageInfo = JSON.parse(
   fs.readFileSync(path.join(__dirname, "package.json"))
 );
 
-const { SECRETS_PATH } = process.env;
-const SecretIds = {
-  index: `${SECRETS_PATH}/infrastructure/index`,
-  meadow: "config/meadow",
-};
-let Initialized = false;
-let Secrets = {};
-
-async function initialize() {
-  if (Initialized) return;
-
-  const client = new SecretsManagerClient();
-  for (const source in SecretIds) {
-    const SecretId = SecretIds[source];
-    console.debug("loading", SecretId, "from", source);
-    const cmd = new GetSecretValueCommand({ SecretId });
-    const { SecretString } = await client.send(cmd);
-    Secrets[source] = JSON.parse(SecretString);
-  }
-  Initialized = true;
-  return Secrets;
-}
-
 function apiToken() {
   const token = {
     displayName: ["Digital Collection API v2"],
@@ -92,7 +65,6 @@ module.exports = {
   dcApiEndpoint,
   dcUrl,
   devTeamNetIds,
-  initialize,
   openSearchEndpoint,
   prefix,
   region,

api/src/handlers/get-file-set-download.js

@@ -69,7 +69,8 @@ exports.handler = wrap(async (event, context) => {
 async function loadSecrets(client) {
   if (Secrets) return Secrets;
 
-  const { SECRETS_PATH } = process.env;
+  const SECRETS_PATH =
+    process.env?.API_CONFIG_PREFIX || process.env.SECRETS_PATH;
   const SecretId = `${SECRETS_PATH}/config/av-download`;
   try {
     const cmd = new GetSecretValueCommand({ SecretId });

api/src/handlers/middleware.js

@@ -19,14 +19,16 @@ const debug = require("debug")("api.middleware");
 const Honeybadger = require("../honeybadger-setup");
 const { StatusCodes } = require("http-status-codes");
 const { SECRETS_PATH } = process.env;
+const API_CONFIG_PREFIX = process.env?.API_CONFIG_PREFIX || SECRETS_PATH;
 const SecretPaths = [
-  `${SECRETS_PATH}/config/dcapi`,
+  `${API_CONFIG_PREFIX}/config/dcapi`,
   `${SECRETS_PATH}/infrastructure/index`,
   `${SECRETS_PATH}/infrastructure/nusso`,
 ];
 
 const wrap = function (handler) {
   return async (event, context) => {
+    console.log("getSearch event", JSON.stringify(event));
     await _initializeEnvironment();
 
     let response;