Skip to content

Commit 1d213be

Browse files
mbkleinmbklein
committed
Fix start-with-step to allow av-download stack to run
1 parent 4841a86 commit 1d213be

File tree

4 files changed

+59
-15
lines changed

4 files changed

+59
-15
lines changed

‎Makefile

+2-1
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ serve-https: SERVE_PARAMS = --port 3002 --ssl-cert-file $$HOME/.dev_cert/dev.rdc
9292
serve-https: serve-http
9393
serve: serve-https
9494
start-with-step: deps-node env.json
95+
export AWS_DEFAULT_REGION=us-east-1 ;\
9596
sam local start-lambda -t av-download/template.yaml --host 0.0.0.0 --port 3005 --env-vars $$PWD/env.json --log-file lambda.log & \
9697
echo $$! > .sam-pids ;\
9798
sg open all 3005 ;\
@@ -107,7 +108,7 @@ start-with-step: deps-node env.json
107108
state-machine:
108109
export TEMPLATE_DIR=$$(mktemp -d); \
109110
yq -o=json '.Resources.avDownloadStateMachine.Properties.Definition' av-download/template.yaml > $$TEMPLATE_DIR/av_download.json; \
110-
aws stepfunctions create-state-machine --endpoint http://localhost:8083 --definition file://$$TEMPLATE_DIR/av_download.json --name "hlsStitcherStepFunction" --role-arn arn:aws:iam::012345678901:role/DummyRole
111+
aws stepfunctions create-state-machine --endpoint http://localhost:8083 --definition file://$$TEMPLATE_DIR/av_download.json --name "hlsStitcherStepFunction" --role-arn arn:aws:iam::012345678901:role/DummyRole --no-cli-pager
111112
deps: deps-node deps-python
112113
style: style-node style-python
113114
test: test-node test-python

‎api/src/handlers/get-file-set-download.js

+20-13
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@ const opensearchResponse = require("../api/response/opensearch");
1717
const path = require("path");
1818

1919
let Secrets;
20+
21+
const getSecret = (key) => {
22+
return process.env[key.toUpperCase()] || Secrets[key];
23+
};
24+
2025
/**
2126
* Handler for download file set endpoint
2227
*/
@@ -136,7 +141,7 @@ function derivativeKey(doc) {
136141

137142
async function getDownloadLink(doc) {
138143
const clientParams = {};
139-
const bucket = Secrets.pyramid_bucket;
144+
const bucket = getSecret("pyramid_bucket");
140145
const key = derivativeKey(doc);
141146

142147
const getObjectParams = {
@@ -200,16 +205,16 @@ const IIIFImageRequest = async (doc) => {
200205
};
201206

202207
async function processAVDownload(doc, email, referer) {
203-
const stepFunctionConfig = Secrets.step_function_endpoint
204-
? { endpoint: Secrets.step_function_endpoint }
208+
const stepFunctionConfig = getSecret("step_function_endpoint")
209+
? { endpoint: getSecret("step_function_endpoint") }
205210
: {};
206211
const client = new SFNClient(stepFunctionConfig);
207212

208213
const fileSet = doc._source;
209214
const url = new URL(fileSet.streaming_url);
210215

211216
const sourceLocation = s3Location(fileSet.streaming_url);
212-
const destinationBucket = Secrets.media_convert_destination_bucket;
217+
const destinationBucket = getSecret("media_convert_destination_bucket");
213218
const fileSetId = path.parse(url.pathname).name;
214219
const fileSetLabel = fileSet.label;
215220
const workId = fileSet.work_id;
@@ -224,14 +229,16 @@ async function processAVDownload(doc, email, referer) {
224229
const filename = isAudio(doc) ? `${fileSetId}.mp3` : `${fileSetId}.mp4`;
225230

226231
var params = {
227-
stateMachineArn: Secrets.av_download_state_machine_arn,
232+
stateMachineArn: getSecret("av_download_state_machine_arn"),
228233
input: JSON.stringify({
229234
configuration: {
230-
startAudioTranscodeFunction: Secrets.start_audio_transcode_function,
231-
startTranscodeFunction: Secrets.start_transcode_function,
232-
transcodeStatusFunction: Secrets.transcode_status_function,
233-
getDownloadLinkFunction: Secrets.get_download_link_function,
234-
sendTemplatedEmailFunction: Secrets.send_templated_email_function,
235+
startAudioTranscodeFunction: getSecret(
236+
"start_audio_transcode_function"
237+
),
238+
startTranscodeFunction: getSecret("start_transcode_function"),
239+
transcodeStatusFunction: getSecret("transcode_status_function"),
240+
getDownloadLinkFunction: getSecret("get_download_link_function"),
241+
sendTemplatedEmailFunction: getSecret("send_templated_email_function"),
235242
},
236243
transcodeInput: {
237244
settings: settings,
@@ -248,8 +255,8 @@ async function processAVDownload(doc, email, referer) {
248255
},
249256
sendEmailInput: {
250257
to: email,
251-
template: Secrets.av_download_email_template,
252-
from: Secrets.repository_email,
258+
template: getSecret("av_download_email_template"),
259+
from: getSecret("repository_email"),
253260
params: {
254261
downloadLink: "",
255262
fileSetId,
@@ -280,7 +287,7 @@ async function processAVDownload(doc, email, referer) {
280287

281288
function s3Location(streaming_url) {
282289
const url = new URL(streaming_url);
283-
return `s3://${Secrets.streaming_bucket}${url.pathname}`;
290+
return `s3://${getSecret("streaming_bucket")}${url.pathname}`;
284291
}
285292

286293
function invalidRequest(code, message) {

‎api/template.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,23 @@ Resources:
359359
Description: Downloads a file set.
360360
#* Layers:
361361
#* - !Ref apiDependencies
362+
Environment:
363+
Variables:
364+
AV_DOWNLOAD_EMAIL_TEMPLATE: !Ref AWS::NoValue
365+
AV_DOWNLOAD_STATE_MACHINE_ARN: !Ref AWS::NoValue
366+
GET_DOWNLOAD_LINK_FUNCTION: !Ref AWS::NoValue
367+
MEDIA_CONVERT_DESTINATION_BUCKET: !Ref AWS::NoValue
368+
MEDIA_CONVERT_ENDPOINT: !Ref AWS::NoValue
369+
MEDIA_CONVERT_JOB_QUEUE_ARN: !Ref AWS::NoValue
370+
MEDIA_CONVERT_ROLE_ARN: !Ref AWS::NoValue
371+
PYRAMID_BUCKET: !Ref AWS::NoValue
372+
REPOSITORY_EMAIL: !Ref AWS::NoValue
373+
SEND_TEMPLATED_EMAIL_FUNCTION: !Ref AWS::NoValue
374+
START_AUDIO_TRANSCODE_FUNCTION: !Ref AWS::NoValue
375+
START_TRANSCODE_FUNCTION: !Ref AWS::NoValue
376+
STEP_FUNCTION_ENDPOINT: !Ref AWS::NoValue
377+
STREAMING_BUCKET: !Ref AWS::NoValue
378+
TRANSCODE_STATUS_FUNCTION: !Ref AWS::NoValue
362379
Policies:
363380
- !Ref SecretsPolicy
364381
- Version: 2012-10-17

‎bin/make_env.sh

+20-1
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,14 @@ get_secret() {
1212
fi
1313
}
1414

15+
aws_account_id=$(aws sts get-caller-identity --query 'Account' --output text)
16+
media_convert_endpoint=$(aws mediaconvert describe-endpoints --query 'Endpoints[0].Url' --output text)
17+
media_convert_queue=$(aws mediaconvert get-queue --name Default --query Queue.Name --output text)
1518

1619
cat <<EOF > env.json
1720
{
1821
"Parameters": {
22+
"AWS_REGION": "us-east-1",
1923
"API_TOKEN_NAME": "dcApiLocal",
2024
"API_TOKEN_SECRET": "$(get_secret staging/config/dcapi api_token_secret)",
2125
"DC_API_ENDPOINT": "https://${DEV_PREFIX}.dev.rdc.library.northwestern.edu:3002",
@@ -24,7 +28,22 @@ cat <<EOF > env.json
2428
"DEV_TEAM_NET_IDS": "$(aws ec2 describe-tags --filters "Name=resource-id,Values=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" "Name=key,Values=NetID" --query 'Tags[0].Value' --output text)",
2529
"ENV_PREFIX": "${DEV_PREFIX}-${DEV_ENV}",
2630
"READING_ROOM_IPS": "",
27-
"SECRETS_PATH": "dev-environment"
31+
"SECRETS_PATH": "dev-environment",
32+
"AV_DOWNLOAD_EMAIL_TEMPLATE": "av-download-template",
33+
"AV_DOWNLOAD_STATE_MACHINE_ARN": "arn:aws:states:us-east-1:123456789012:stateMachine:hlsStitcherStepFunction",
34+
"GET_DOWNLOAD_LINK_FUNCTION": "arn:aws:lambda:us-east-1:123456789012:function:getDownloadLinkFunction",
35+
"MEDIA_CONVERT_DESTINATION_BUCKET": "${DEV_PREFIX}-${DEV_ENV}-streaming",
36+
"MEDIA_CONVERT_ENDPOINT": "${media_convert_endpoint}",
37+
"MEDIA_CONVERT_JOB_QUEUE_ARN": "${media_convert_queue}",
38+
"MEDIA_CONVERT_ROLE_ARN": "arn:aws:iam::${aws_account_id}:role/service-role/MediaConvert_Default_Role",
39+
"PYRAMID_BUCKET": "${DEV_PREFIX}-${DEV_ENV}-pyramids",
40+
"REPOSITORY_EMAIL": "repository@northwestern.edu",
41+
"SEND_TEMPLATED_EMAIL_FUNCTION": "arn:aws:lambda:us-east-1:123456789012:function:sendTemplatedEmailFunction",
42+
"START_AUDIO_TRANSCODE_FUNCTION": "arn:aws:lambda:us-east-1:123456789012:function:startAudioTranscodeFunction",
43+
"START_TRANSCODE_FUNCTION": "arn:aws:lambda:us-east-1:123456789012:function:startTranscodeFunction",
44+
"STEP_FUNCTION_ENDPOINT": "http://172.17.0.1:8083",
45+
"STREAMING_BUCKET": "${DEV_PREFIX}-${DEV_ENV}-streaming",
46+
"TRANSCODE_STATUS_FUNCTION": "arn:aws:lambda:us-east-1:123456789012:function:transcodeStatusFunction"
2847
}
2948
}
3049
EOF

0 commit comments

Comments
 (0)