Skip to content

Commit b3f30ae

Browse files
mbkleinmbklein
committed
Refactor project structure into sub-stacks
Add ability to selectively deploy stacks Use cfn-lint instead of sam validate Add DEFAULT_SEARCH_SIZE parameter Add make features for env.json and samconfig.yaml and update documentation Fix issues with templates & makefile and update documentation
1 parent 167ae10 commit b3f30ae

File tree

166 files changed

+1804
-1944
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

166 files changed

+1804
-1944
lines changed

‎.github/workflows/deploy.yml

+6-9
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,17 @@ jobs:
2020
contents: read
2121
environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}
2222
steps:
23-
- name: Set CONFIG_ENV from Branch Name
23+
- name: Set DEPLOY_ENV from Branch Name
2424
run: |
2525
if [[ $BRANCH == 'refs/heads/main' ]]; then
26-
echo "CONFIG_ENV=production" >> $GITHUB_ENV
26+
echo "DEPLOY_ENV=production" >> $GITHUB_ENV
2727
else
28-
echo "CONFIG_ENV=$(echo $BRANCH | awk -F/ '{print $NF}')" >> $GITHUB_ENV
28+
echo "DEPLOY_ENV=$(echo $BRANCH | awk -F/ '{print $NF}')" >> $GITHUB_ENV
2929
fi
3030
env:
3131
BRANCH: ${{ github.ref }}
3232
- name: Confirm deploy environment
33-
run: echo "Deploying to '$CONFIG_ENV' environment"
33+
run: echo "Deploying to '$DEPLOY_ENV' environment"
3434
- name: Set GitHub Deploy Key
3535
uses: webfactory/ssh-agent@v0.5.3
3636
with:
@@ -49,16 +49,13 @@ jobs:
4949
with:
5050
role-to-assume: arn:aws:iam::${{ secrets.AwsAccount }}:role/github-actions-role
5151
aws-region: us-east-1
52-
- run: ln -s .tfvars/dc-api/samconfig.toml .
53-
- run: ln -s .tfvars/dc-api/$CONFIG_ENV.parameters .
52+
- run: ln -s .tfvars/dc-api/samconfig.${DEPLOY_ENV}.yaml .
5453
- run: make build
5554
- run: |
5655
sam deploy \
5756
--no-confirm-changeset \
5857
--no-fail-on-empty-changeset \
59-
--config-env $CONFIG_ENV \
60-
--config-file ./samconfig.toml \
61-
--parameter-overrides $(while IFS='=' read -r key value; do params+=" $key=$value"; done < ./$CONFIG_ENV.parameters && echo "$params HoneybadgerRevision=$HONEYBADGER_REVISION") \
58+
--config-file ./samconfig.${DEPLOY_ENV}.yaml \
6259
| sed 's/\(Parameter overrides\s*\): .*/\1: ***** REDACTED *****/'
6360
exit ${PIPESTATUS[0]}
6461
env:

‎.github/workflows/test-node.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ on:
33
push:
44
paths:
55
- ".github/workflows/test-node.yml"
6-
- "node/**"
6+
- "api/**"
77
workflow_dispatch:
88
defaults:
99
run:
10-
working-directory: ./node
10+
working-directory: ./api
1111
jobs:
1212
test:
1313
runs-on: ubuntu-latest
@@ -20,7 +20,7 @@ jobs:
2020
with:
2121
node-version: 20.x
2222
cache: "npm"
23-
cache-dependency-path: 'node/package-lock.json'
23+
cache-dependency-path: 'api/package-lock.json'
2424
- run: npm ci
2525
- name: Check code style
2626
run: npm run lint && npm run prettier

‎.github/workflows/validate-template.yml

+5-3
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,15 @@ jobs:
1616
- uses: actions/setup-python@v4
1717
with:
1818
python-version: '3.12'
19+
- name: Install cfn-lint
20+
run: pip install cfn-lint
1921
- uses: aws-actions/setup-sam@v1
20-
- name: sam fix https://github.com/aws/aws-sam-cli/issues/4527
21-
run: $(dirname $(readlink $(which sam)))/pip install --force-reinstall "cryptography==38.0.4"
22+
# - name: sam fix https://github.com/aws/aws-sam-cli/issues/4527
23+
# run: $(dirname $(readlink $(which sam)))/pip install --force-reinstall "cryptography==38.0.4"
2224
- uses: aws-actions/configure-aws-credentials@master
2325
with:
2426
role-to-assume: arn:aws:iam::${{ secrets.AwsAccount }}:role/github-actions-role
2527
aws-region: us-east-1
2628
- uses: actions/checkout@v3
2729
- name: Validate template
28-
run: sam build && sam validate
30+
run: make build && make validate

‎.husky/pre-commit

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
#!/usr/bin/env sh
22
. "$(dirname -- "$0")/_/husky.sh"
3-
cd node && npm run lint && npm run prettier && cd -
3+
cd api && npm run lint && npm run prettier && cd -
44
cd chat/src && ruff check . && cd -

‎.tool-versions

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
nodejs 20.15.0
22
java corretto-19.0.1.10.1
3-
aws-sam-cli 1.107.0
3+
aws-sam-cli 1.135.0
44
python 3.12.2

‎Makefile

+83-45
Original file line numberDiff line numberDiff line change
@@ -5,50 +5,67 @@ ENV=dev
55
SHELL := /bin/bash
66

77
help:
8-
echo "make build | build the SAM project"
9-
echo "make serve | alias for serve-https"
10-
echo "make clean | remove all installed dependencies and build artifacts"
11-
echo "make deps | install all dependencies"
12-
echo "make link | create hard links to allow for hot reloading of a built project"
13-
echo "make secrets | symlink secrets files from ../tfvars"
14-
echo "make start-with-step | run the SAM server locally with step function & download lambdas"
15-
echo "make style | run all style checks"
16-
echo "make test | run all tests"
17-
echo "make cover | run all tests with coverage"
18-
echo "make env ENV=[env] | activate env.\$$ENV.json file (default: dev)"
19-
echo "make deps-node | install node dependencies"
20-
echo "make deps-python | install python dependencies"
21-
echo "make serve-http | run the SAM server locally (HTTP on port 3000)"
22-
echo "make serve-https | run the SAM server locally (HTTPS on port 3002)"
23-
echo "make style-node | run node code style check"
24-
echo "make style-python | run python code style check"
25-
echo "make test-node | run node tests"
26-
echo "make test-python | run python tests"
27-
echo "make cover-node | run node tests with coverage"
28-
echo "make cover-python | run python tests with coverage"
29-
.aws-sam/build.toml: ./template.yaml node/package-lock.json node/src/package-lock.json chat/dependencies/requirements.txt chat/src/requirements.txt
30-
sed -Ei.orig 's/^(\s+)#\*\s/\1/' template.yaml
31-
sed -Ei.orig 's/^(\s+)#\*\s/\1/' chat/template.yaml
32-
sam build --cached --parallel
33-
mv template.yaml.orig template.yaml
34-
mv chat/template.yaml.orig chat/template.yaml
35-
deps-node:
36-
cd node/src ;\
8+
echo "make build | build the SAM project"
9+
echo "make serve | alias for serve-https"
10+
echo "make clean | remove all installed dependencies and build artifacts"
11+
echo "make deps | install all dependencies"
12+
echo "make env.json | create an env.json file for the current user's environment"
13+
echo "make link | create hard links to allow for hot reloading of a built project"
14+
echo "make secrets | symlink secrets files from ../tfvars"
15+
echo "make start-with-step | run the SAM server locally with step function & download lambdas"
16+
echo "make style | run all style checks"
17+
echo "make test | run all tests"
18+
echo "make cover | run all tests with coverage"
19+
echo "make env ENV=[env] | activate env.\$$ENV.json file (default: dev)"
20+
echo "make deps-node | install node dependencies"
21+
echo "make deps-python | install python dependencies"
22+
echo "make samconfig.NAME.yaml" | create a user samconfig file for the specified username"
23+
echo "make build | build the SAM project for deploying"
24+
echo "make deploy | deploy the SAM project to AWS"
25+
echo "make sync | sync the SAM project to AWS for quick development"
26+
echo "make sync-code | sync the SAM project to AWS (code changes only)"
27+
echo "make serve-http | run the SAM server locally (HTTP on port 3000)"
28+
echo "make serve-https | run the SAM server locally (HTTPS on port 3002)"
29+
echo "make style-node | run node code style check"
30+
echo "make style-python | run python code style check"
31+
echo "make test-node | run node tests"
32+
echo "make test-python | run python tests"
33+
echo "make cover-node | run node tests with coverage"
34+
echo "make cover-python | run python tests with coverage"
35+
36+
.aws-sam/build.toml: ./template.yaml api/package-lock.json api/src/package-lock.json chat/dependencies/requirements.txt chat/src/requirements.txt
37+
sed -Ei.orig 's/"dependencies"/"devDependencies"/' api/src/package.json
38+
cp api/src/package-lock.json api/src/package-lock.json.orig
39+
cd api/src && npm i --package-lock-only && cd -
40+
for d in . api av-download chat docs ; do \
41+
sed -Ei.orig 's/^(\s+)#\*\s/\1/' $$d/template.yaml; \
42+
done
43+
44+
-sam build --cached --parallel
45+
46+
for d in . api av-download chat docs ; do \
47+
mv $$d/template.yaml.orig $$d/template.yaml; \
48+
done
49+
mv api/src/package.json.orig api/src/package.json
50+
mv api/src/package-lock.json.orig api/src/package-lock.json
51+
deps-api:
52+
cd api/src ;\
3753
npm list >/dev/null 2>&1 ;\
3854
src_deps=$$? ;\
3955
cd .. ;\
4056
npm list >/dev/null 2>&1 ;\
4157
dev_deps=$$? ;\
4258
test $$src_deps -eq 0 -a $$dev_deps -eq 0 || npm ci
43-
44-
cd lambdas ;\
59+
deps-av-download:
60+
cd av-download/lambdas ;\
4561
npm list >/dev/null 2>&1 || npm ci
62+
deps-node: deps-api deps-av-download
4663
cover-node: deps-node
47-
cd node && npm run test:coverage
64+
cd api && npm run test:coverage
4865
style-node: deps-node
49-
cd node && npm run prettier
66+
cd api && npm run prettier
5067
test-node: deps-node
51-
cd node && npm run test
68+
cd api && npm run test
5269
deps-python:
5370
cd chat/src && pip install -r requirements.txt && pip install -r requirements-dev.txt
5471
cover-python: deps-python
@@ -63,30 +80,51 @@ test-python: deps-python
6380
cd chat && pytest
6481
python-version:
6582
cd chat && python --version
66-
build: .aws-sam/build.toml
67-
serve-http: deps-node
83+
build:
84+
bash -c "trap 'trap - SIGINT SIGTERM ERR; $(MAKE) reset; exit 1' SIGINT SIGTERM ERR; $(MAKE) .aws-sam/build.toml reset"
85+
validate:
86+
cfn-lint template.yaml **/template.yaml --ignore-checks E3510 W1028 W8001
87+
serve-http: deps-node env.json
6888
@printf '\033[0;31mWARNING: Serving only the local HTTP API. The chat websocket API is not available in local mode.\033[0m\n'
6989
rm -rf .aws-sam
70-
sam local start-api --host 0.0.0.0 --log-file dc-api.log ${SERVE_PARAMS}
90+
sam local start-api -t api/template.yaml --env-vars $$PWD/env.json --host 0.0.0.0 --log-file dc-api.log ${SERVE_PARAMS}
7191
serve-https: SERVE_PARAMS = --port 3002 --ssl-cert-file $$HOME/.dev_cert/dev.rdc.cert.pem --ssl-key-file $$HOME/.dev_cert/dev.rdc.key.pem
7292
serve-https: serve-http
7393
serve: serve-https
74-
start-with-step: deps-node
75-
sam local start-lambda --host 0.0.0.0 --port 3005 --env-vars env.json --log-file lambda.log & \
94+
start-with-step: deps-node env.json
95+
sam local start-lambda -t av-download/template.yaml --host 0.0.0.0 --port 3005 --env-vars $$PWD/env.json --log-file lambda.log & \
7696
echo $$! > .sam-pids ;\
77-
sam local start-api --host 0.0.0.0 --port 3002 --log-file dc-api.log \
97+
sg open all 3005 ;\
98+
sam local start-api -t api/template.yaml --env-vars $$PWD/env.json --host 0.0.0.0 --port 3002 --log-file dc-api.log \
7899
--ssl-cert-file $$HOME/.dev_cert/dev.rdc.cert.pem --ssl-key-file $$HOME/.dev_cert/dev.rdc.key.pem & \
79100
echo $$! >> .sam-pids ;\
80101
docker run --rm -p 8083:8083 -e LAMBDA_ENDPOINT=http://172.17.0.1:3005/ amazon/aws-stepfunctions-local ;\
102+
echo -n "Shutting down..." ;\
103+
sg close all 3005 ;\
81104
kill $$(cat .sam-pids) ;\
82-
rm -f .sam-pids
105+
rm -f .sam-pids ;\
106+
echo ""
107+
state-machine:
108+
export TEMPLATE_DIR=$$(mktemp -d); \
109+
yq -o=json '.Resources.avDownloadStateMachine.Properties.Definition' av-download/template.yaml > $$TEMPLATE_DIR/av_download.json; \
110+
aws stepfunctions create-state-machine --endpoint http://localhost:8083 --definition file://$$TEMPLATE_DIR/av_download.json --name "hlsStitcherStepFunction" --role-arn arn:aws:iam::012345678901:role/DummyRole
83111
deps: deps-node deps-python
84112
style: style-node style-python
85113
test: test-node test-python
86114
cover: cover-node cover-python
87-
env:
88-
ln -fs ./env.${ENV}.json ./env.json
115+
env.json:
116+
./bin/make_env.sh
117+
samconfig.%.yaml:
118+
DEV_PREFIX=$* ./bin/make_deploy_config.sh
119+
deploy: build samconfig.$(DEV_PREFIX).yaml
120+
sam deploy --config-file samconfig.$(DEV_PREFIX).yaml --stack-name dc-api-$(DEV_PREFIX)
121+
sync: samconfig.$(DEV_PREFIX).yaml
122+
sam sync --config-file samconfig.$(DEV_PREFIX).yaml --stack-name dc-api-$(DEV_PREFIX) --watch $(ARGS)
123+
sync-code: ARGS=--code
124+
sync-code: sync
89125
secrets:
90-
ln -s ../tfvars/dc-api/* .
126+
ln -s ../tfvars/dc-api/*.yaml .
91127
clean:
92-
rm -rf .aws-sam node/node_modules node/src/node_modules python/**/__pycache__ python/.coverage python/.ruff_cache
128+
rm -rf .aws-sam api/.aws-sam chat/.aws-sam av-download/.aws-sam api/node_modules api/src/node_modules chat/**/__pycache__ chat/.coverage chat/.ruff_cache
129+
reset:
130+
for f in $$(find . -maxdepth 2 -name '*.orig'); do mv $$f $${f%%.orig}; done

0 commit comments

Comments
 (0)