Use ffmpeg build for Amazon Linux 2023

Use @aws-sdk/lib-storage to upload transcoded audio to S3
Make sure ffmpeg exits cleanly

Author: mbklein

.gitignore

@@ -231,3 +231,4 @@ env.*.json
 *.parameters
 /schemas
 .sam-pids
+av-download/layers/ffmpeg/bin

Makefile

@@ -126,6 +126,6 @@ sync-code: sync
 secrets:
 	ln -s ../tfvars/dc-api/*.yaml .
 clean:
-	rm -rf .aws-sam api/.aws-sam chat/.aws-sam av-download/.aws-sam api/node_modules api/src/node_modules chat/**/__pycache__ chat/.coverage chat/.ruff_cache
+	rm -rf .aws-sam api/.aws-sam chat/.aws-sam av-download/.aws-sam api/node_modules api/src/node_modules av-download/layers chat/**/__pycache__ chat/.coverage chat/.ruff_cache
 reset:
 	for f in $$(find . -maxdepth 2 -name '*.orig'); do mv $$f $${f%%.orig}; done

api/template.yaml

@@ -13,7 +13,7 @@ Description: dc-api-v2 API
 Globals:
   Function:
     CodeUri: ./src
-    Runtime: nodejs20.x
+    Runtime: nodejs22.x
     Architectures:
       - x86_64
     MemorySize: 128
@@ -130,10 +130,10 @@ Resources:
   #*     Description: Dependencies for API handlers
   #*     ContentUri: ./dependencies
   #*     CompatibleRuntimes:
-  #*       - nodejs20.x
+  #*       - nodejs22.x
   #*     LicenseInfo: Apache-2.0
   #*   Metadata:
-  #*     BuildMethod: nodejs20.x
+  #*     BuildMethod: nodejs22.x
   # Configuration
   apiConfiguration:
     Type: AWS::SecretsManager::Secret

av-download/lambdas/package-lock.json

@@ -15,6 +15,6 @@
     "aws-sdk": "^2.1640.0"
   },
   "dependencies": {
-    "fluent-ffmpeg": "2.1.2"
+    "fluent-ffmpeg": "2.1.3"
   }
 }

av-download/lambdas/package.json

@@ -1,41 +1,56 @@
-const AWS = require('aws-sdk');
+const { S3Client, PutObjectCommand } = require('@aws-sdk/client-s3');
+const { Upload } = require('@aws-sdk/lib-storage');
 const ffmpeg = require('fluent-ffmpeg');
 const stream = require('stream');
 
+
 module.exports.handler = async (event) => {
-  
-  const s3 = new AWS.S3();
+  const s3Client = new S3Client();
   const url = event.streamingUrl;
   const referer = event.referer || null;
   const inputOptions = referer ? ["-referer", referer] : [];
   const bucket = event.destinationBucket;
   const key = event.destinationKey;
   const pass = new stream.PassThrough();
+  const upload = new Upload({
+    client: s3Client,
+    params: {
+      Bucket: bucket,
+      Key: key,
+      Body: pass,
+      ContentType: 'audio/mp3'
+    }
+  });
+
+  upload.on("end", ({ loaded, total, part, Key, Bucket }) => {
+    console.log(`Uploaded ${total} bytes in ${part} parts to s3://${Bucket}/${Key}`);
+  });
 
-  return new Promise((resolve, reject) => {
-    ffmpeg()
+  try {
+    ffmpeg({ logger: console })
       .input(url)
       .inputOptions(inputOptions)
-      .format('mp3')
-      .output(pass, { end: true })
-      .on('start', () => {
-        s3.upload({
-          Bucket: bucket,
-          Key: key,
-          Body: pass,
-        }, (error, _data) => {
-          if (error) {
-            console.error('upload failed', error);
-            reject(error);
-          } else {
-            resolve({ success: true });
-          }
-        });
+      .format("mp3")
+      .output(pass, { end: false })
+      .on('start', function(commandLine) {
+        console.log('Spawned Ffmpeg with command: ' + commandLine);
       })
-      .on('error', (error) => {
-        console.error('ffmpeg error', error);
-        reject(error);
+      .on("error", (error) => {
+        console.error("ffmpeg error", error);
+      })
+      .on("end", () => {
+        console.log("ffmpeg finished");
+        pass.end();
+      })
+      .on("*", (event, ...args) => {
+        console.log('ffmpeg fired event:', event, 'with', args);
       })
       .run();
-  });
+  } catch (error) {
+    console.error("ffmpeg error", error);
+    reject(error);
+  }
+
+  await upload.done();
+  return { status: 200, message: 'done' };
 }

av-download/lambdas/start-audio-transcode.js

@@ -14,7 +14,7 @@ Parameters:
   FfmpegLayer:
     Type: String
     Description: "FFMPEG Lambda Layer ARN"
-    Default: "arn:aws:lambda:us-east-1:625046682746:layer:ffmpeg:11"
+    Default: arn:aws:lambda:us-east-1:625046682746:layer:ffmpeg:15
   MediaConvertDestinationBucket:
     Type: String
     Description: S3 bucket destination for transcoded AV resource

av-download/template.yaml

@@ -263,7 +263,7 @@ Resources:
           - bedrock:InvokeModelWithResponseStream
           Resource: "*"
     #* Metadata:
-    #*   BuildMethod: nodejs20.x
+    #*   BuildMethod: nodejs22.x
   ChatSyncFunction:
     Type: AWS::Serverless::Function
     Properties:
@@ -322,7 +322,7 @@ Resources:
           - bedrock:InvokeModelWithResponseStream
           Resource: "*"
     #* Metadata:
-    #*   BuildMethod: nodejs20.x
+    #*   BuildMethod: nodejs22.x
   ChatMetricsLog:
     Type: AWS::Logs::LogGroup
     Properties:

chat/template.yaml

@@ -18,7 +18,7 @@ Resources:
   rootRedirect:
     Type: AWS::Serverless::Function
     Properties:
-      Runtime: nodejs20.x
+      Runtime: nodejs22.x
       CodeUri: ./redirect
       Handler: index.handler
       Timeout: 1

docs/template.yaml

@@ -55,10 +55,6 @@ Parameters:
   DevTeamNetIds:
     Type: String
     Description: Northwestern NetIDs of the development team
-  FfmpegLayer:
-    Type: String
-    Description: "FFMPEG Lambda Layer ARN"
-    Default: "arn:aws:lambda:us-east-1:625046682746:layer:ffmpeg:11"
   EnvironmentPrefix:
     Type: String
     Description: Index Prefix
@@ -177,7 +173,6 @@ Resources:
     Properties:
       Location: ./av-download/template.yaml
       Parameters:
-        FfmpegLayer: !Ref FfmpegLayer
         MediaConvertDestinationBucket: !Ref MediaConvertDestinationBucket
         MediaConvertEndpoint: !Ref MediaConvertEndpoint
         MediaConvertJobQueueArn: !Ref MediaConvertJobQueueArn