From 3840a4ad4235406972585ebc9192b17229315277 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Sun, 10 May 2020 23:42:03 +0200 Subject: [PATCH 01/53] feat(lambda): docker code Add `Code.fromDockerImage` and `Code.fromDockerAsset` that offer a generic way of working with Docker for Lambda code. `Code.fromDockerImage`: run a command in an existing Docker image `Code.fromDockerAsset`: build an image and then run a command in it Both `Code` classes take an `assetPath` prop that corresponds to the path of the asset directory that will contain the build output of the Docker container. This path is automatically mounted at `/asset` in the container. Using a combination of image and command, the container is then responsible for putting content at this location. Additional volumes can be mounted if needed. This will allow to refactor `aws-lambda-nodejs` and create other language specific modules. --- packages/@aws-cdk/aws-lambda/lib/code.ts | 157 ++++++++++++++++++ .../test/integ.docker.expected.json | 103 ++++++++++++ .../@aws-cdk/aws-lambda/test/integ.docker.ts | 27 +++ .../test/python-lambda-handler/index.py | 5 + .../python-lambda-handler/requirements.txt | 1 + .../@aws-cdk/aws-lambda/test/test.code.ts | 85 ++++++++++ 6 files changed, 378 insertions(+) create mode 100644 packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json create mode 100644 packages/@aws-cdk/aws-lambda/test/integ.docker.ts create mode 100644 packages/@aws-cdk/aws-lambda/test/python-lambda-handler/index.py create mode 100644 packages/@aws-cdk/aws-lambda/test/python-lambda-handler/requirements.txt diff --git a/packages/@aws-cdk/aws-lambda/lib/code.ts b/packages/@aws-cdk/aws-lambda/lib/code.ts index f9df534834195..b4556da4ccd95 100644 --- a/packages/@aws-cdk/aws-lambda/lib/code.ts +++ b/packages/@aws-cdk/aws-lambda/lib/code.ts @@ -1,6 +1,8 @@ import * as s3 from '@aws-cdk/aws-s3'; import * as s3_assets from '@aws-cdk/aws-s3-assets'; import * as cdk from '@aws-cdk/core'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; export abstract class Code { /** @@ -43,6 +45,20 @@ export abstract class Code { return new AssetCode(path, options); } + /** + * Lambda code from a command run in an existing Docker image. + */ + public static fromDockerImage(options: DockerImageCodeOptions): AssetCode { + return new DockerImageCode(options); + } + + /** + * Lambda code from a command run in a Docker image built from a Dockerfile. + */ + public static fromDockerAsset(options: DockerAssetCodeOptions): AssetCode { + return new DockerAssetCode(options); + } + /** * @deprecated use `fromAsset` */ @@ -206,6 +222,143 @@ export class AssetCode extends Code { } } +/** + * A Docker volume + */ +export interface DockerVolume { + /** + * The path to the file or directory on the host machine + */ + readonly hostPath: string; + + /** + * The path where the file or directory is mounted in the container + */ + readonly containerPath: string; +} + +/** + * Docker run options + */ +export interface DockerRunOptions { + /** + * The command to run in the container. + */ + readonly command: string[]; + + /** + * The path of the asset directory that will contain the build output of the Docker + * container. This path is mounted at `/asset` in the container. It is created + * if it doesn't exist. + */ + readonly assetPath: string; + + /** + * Additional Docker volumes to mount. + * + * @default - no additional volumes are mounted + */ + readonly volumes?: DockerVolume[]; +} + +/** + * Options for DockerImageCode + */ +export interface DockerImageCodeOptions extends DockerRunOptions { + /** + * The Docker image where the command will run. + */ + readonly image: string; +} + +/** + * Lambda code from a command run in an existing Docker image + */ +export class DockerImageCode extends AssetCode { + constructor(options: DockerImageCodeOptions) { + if (!fs.existsSync(options.assetPath)) { + fs.mkdirSync(options.assetPath); + } + + const volumes = options.volumes || []; + + const dockerArgs: string[] = [ + 'run', '--rm', + '-v', `${options.assetPath}:/asset`, + ...flatten(volumes.map(v => ['-v', `${v.hostPath}:${v.containerPath}`])), + options.image, + ...options.command, + ]; + + const docker = spawnSync('docker', dockerArgs); + + if (docker.error) { + throw docker.error; + } + + if (docker.status !== 0) { + throw new Error(`[Status ${docker.status}] stdout: ${docker.stdout?.toString().trim()}\n\n\nstderr: ${docker.stderr?.toString().trim()}`); + } + + super(options.assetPath); + } +} + +/** + * Options for DockerAssetCode + */ +export interface DockerAssetCodeOptions extends DockerRunOptions { + /** + * The path to the directory containing the Docker file. + */ + readonly dockerPath: string; + + /** + * Build args + * + * @default - no build args + */ + readonly buildArgs?: { [key: string]: string }; +} + +/** + * Lambda code from a command run in a Docker image built from a Dockerfile. + */ +export class DockerAssetCode extends DockerImageCode { + constructor(options: DockerAssetCodeOptions) { + const buildArgs = options.buildArgs || {}; + + const dockerArgs: string[] = [ + 'build', + ...flatten(Object.entries(buildArgs).map(([k, v]) => ['--build-arg', `${k}=${v}`])), + options.dockerPath, + ]; + + const docker = spawnSync('docker', dockerArgs); + + if (docker.error) { + throw docker.error; + } + + if (docker.status !== 0) { + throw new Error(`[Status ${docker.status}] stdout: ${docker.stdout?.toString().trim()}\n\n\nstderr: ${docker.stderr?.toString().trim()}`); + } + + const match = docker.stdout.toString().match(/Successfully built ([a-z0-9]+)/); + + if (!match) { + throw new Error('Failed to extract image ID from Docker build output'); + } + + super({ + assetPath: options.assetPath, + command: options.command, + image: match[1], + volumes: options.volumes, + }); + } +} + export interface ResourceBindOptions { /** * The name of the CloudFormation property to annotate with asset metadata. @@ -312,3 +465,7 @@ export class CfnParametersCode extends Code { } } } + +function flatten(x: string[][]) { + return Array.prototype.concat([], ...x); +} diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json b/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json new file mode 100644 index 0000000000000..ed1b35fd53888 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json @@ -0,0 +1,103 @@ +{ + "Resources": { + "FunctionServiceRole675BB04A": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "Function76856677": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParametersb30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8S3BucketB1206B28" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersb30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8S3VersionKeyDD15AE2C" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersb30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8S3VersionKeyDD15AE2C" + } + ] + } + ] + } + ] + ] + } + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "FunctionServiceRole675BB04A", + "Arn" + ] + }, + "Runtime": "python3.6" + }, + "DependsOn": [ + "FunctionServiceRole675BB04A" + ] + } + }, + "Parameters": { + "AssetParametersb30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8S3BucketB1206B28": { + "Type": "String", + "Description": "S3 bucket for asset \"b30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8\"" + }, + "AssetParametersb30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8S3VersionKeyDD15AE2C": { + "Type": "String", + "Description": "S3 key for asset version \"b30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8\"" + }, + "AssetParametersb30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8ArtifactHash9F155131": { + "Type": "String", + "Description": "Artifact hash for asset \"b30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8\"" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts b/packages/@aws-cdk/aws-lambda/test/integ.docker.ts new file mode 100644 index 0000000000000..fd7ca5bbb784a --- /dev/null +++ b/packages/@aws-cdk/aws-lambda/test/integ.docker.ts @@ -0,0 +1,27 @@ +import { App, Construct, Stack, StackProps } from '@aws-cdk/core'; +import * as path from 'path'; +import * as lambda from '../lib'; + +class TestStack extends Stack { + constructor(scope: Construct, id: string, props?: StackProps) { + super(scope, id, props); + + new lambda.Function(this, 'Function', { + code: lambda.Code.fromDockerImage({ + image: 'python:3.6', + assetPath: path.join(__dirname, 'python-lambda-handler'), // this is /asset in the container + command: [ + 'pip3', 'install', + '-r', '/asset/requirements.txt', + '-t', '/asset', + ], + }), + runtime: lambda.Runtime.PYTHON_3_6, + handler: 'index.handler', + }); + } +} + +const app = new App(); +new TestStack(app, 'cdk-integ-lambda-docker'); +app.synth(); diff --git a/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/index.py b/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/index.py new file mode 100644 index 0000000000000..960064a9d5b4e --- /dev/null +++ b/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/index.py @@ -0,0 +1,5 @@ +import requests + +def handler(event, context): + r = requests.get('https://aws.amazon.com') + print(r.status_code) diff --git a/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/requirements.txt b/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/requirements.txt new file mode 100644 index 0000000000000..b4500579db515 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/requirements.txt @@ -0,0 +1 @@ +requests==2.23.0 diff --git a/packages/@aws-cdk/aws-lambda/test/test.code.ts b/packages/@aws-cdk/aws-lambda/test/test.code.ts index 98ac8e0364cac..d7cb353eadc2e 100644 --- a/packages/@aws-cdk/aws-lambda/test/test.code.ts +++ b/packages/@aws-cdk/aws-lambda/test/test.code.ts @@ -1,8 +1,10 @@ import { expect, haveResource, haveResourceLike, ResourcePart } from '@aws-cdk/assert'; import * as cdk from '@aws-cdk/core'; import * as cxapi from '@aws-cdk/cx-api'; +import * as child_process from 'child_process'; import { Test } from 'nodeunit'; import * as path from 'path'; +import * as sinon from 'sinon'; import * as lambda from '../lib'; // tslint:disable:no-string-literal @@ -187,6 +189,89 @@ export = { test.done(); }, }, + + 'lambda.Code.fromDockerImage'(test: Test) { + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + + const dockerAssetPath = 'asset-path'; + const srcPath = 'src-path'; + const command = ['this', 'is', 'a', 'build', 'command']; + + lambda.Code.fromDockerImage({ + assetPath: dockerAssetPath, + image: 'alpine', + volumes: [ + { + hostPath: srcPath, + containerPath: '/src', + }, + ], + command, + }); + + test.ok(spawnSyncStub.calledWith('docker', [ + 'run', '--rm', + '-v', `${dockerAssetPath}:/asset`, + '-v', `${srcPath}:/src`, + 'alpine', + ...command, + ])); + + spawnSyncStub.restore(); + + test.done(); + }, + + 'lambda.Code.fromDockerAsset'(test: Test) { + const imageId = 'abcdef123456'; + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from(`Successfully built ${imageId}`), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + + const dockerPath = 'docker-path'; + const testArg = 'cdk-test'; + const dockerAssetPath = 'asset-path'; + const command = ['this', 'is', 'a', 'build', 'command']; + + lambda.Code.fromDockerAsset({ + dockerPath, + buildArgs: { + TEST_ARG: testArg, + }, + assetPath: dockerAssetPath, + command, + }); + + test.ok(spawnSyncStub.calledWith('docker', [ + 'build', + '--build-arg', `TEST_ARG=${testArg}`, + dockerPath, + ])); + + test.ok(spawnSyncStub.calledWith('docker', [ + 'run', '--rm', + '-v', `${dockerAssetPath}:/asset`, + imageId, + ...command, + ])); + + spawnSyncStub.restore(); + + test.done(); + + }, }; function defineFunction(code: lambda.Code, runtime: lambda.Runtime = lambda.Runtime.NODEJS_10_X) { From ca62f37a7a29f1b7ef68be05a2e1b4a9e5390164 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Sat, 16 May 2020 15:01:57 +0200 Subject: [PATCH 02/53] environment --- packages/@aws-cdk/aws-lambda/lib/code.ts | 9 +++++++++ .../aws-lambda/test/integ.docker.expected.json | 18 +++++++++--------- packages/@aws-cdk/aws-lambda/test/test.code.ts | 6 ++++++ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/code.ts b/packages/@aws-cdk/aws-lambda/lib/code.ts index c6a17f1848708..8c17ac82d246e 100644 --- a/packages/@aws-cdk/aws-lambda/lib/code.ts +++ b/packages/@aws-cdk/aws-lambda/lib/code.ts @@ -260,6 +260,13 @@ export interface DockerRunOptions { * @default - no additional volumes are mounted */ readonly volumes?: DockerVolume[]; + + /** + * The environment variables to pass to the container. + * + * @default - No environment variables. + */ + readonly environment?: { [key: string]: string; }; } /** @@ -282,11 +289,13 @@ export class DockerImageCode extends AssetCode { } const volumes = options.volumes || []; + const environment = options.environment || {}; const dockerArgs: string[] = [ 'run', '--rm', '-v', `${options.assetPath}:/asset`, ...flatten(volumes.map(v => ['-v', `${v.hostPath}:${v.containerPath}`])), + ...flatten(Object.entries(environment).map(([k, v]) => ['--env', `${k}=${v}`])), options.image, ...options.command, ]; diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json b/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json index ed1b35fd53888..9816dcb18319f 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json +++ b/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json @@ -36,7 +36,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParametersb30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8S3BucketB1206B28" + "Ref": "AssetParameters567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9S3BucketDED3100C" }, "S3Key": { "Fn::Join": [ @@ -49,7 +49,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersb30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8S3VersionKeyDD15AE2C" + "Ref": "AssetParameters567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9S3VersionKeyB85BA10F" } ] } @@ -62,7 +62,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersb30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8S3VersionKeyDD15AE2C" + "Ref": "AssetParameters567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9S3VersionKeyB85BA10F" } ] } @@ -87,17 +87,17 @@ } }, "Parameters": { - "AssetParametersb30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8S3BucketB1206B28": { + "AssetParameters567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9S3BucketDED3100C": { "Type": "String", - "Description": "S3 bucket for asset \"b30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8\"" + "Description": "S3 bucket for asset \"567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9\"" }, - "AssetParametersb30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8S3VersionKeyDD15AE2C": { + "AssetParameters567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9S3VersionKeyB85BA10F": { "Type": "String", - "Description": "S3 key for asset version \"b30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8\"" + "Description": "S3 key for asset version \"567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9\"" }, - "AssetParametersb30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8ArtifactHash9F155131": { + "AssetParameters567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9ArtifactHash5913CC34": { "Type": "String", - "Description": "Artifact hash for asset \"b30f71084d0e75786c0a52e418612bc916c98f85291d24847aa53400a0c735e8\"" + "Description": "Artifact hash for asset \"567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9\"" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda/test/test.code.ts b/packages/@aws-cdk/aws-lambda/test/test.code.ts index d7cb353eadc2e..9d797e44b0cc9 100644 --- a/packages/@aws-cdk/aws-lambda/test/test.code.ts +++ b/packages/@aws-cdk/aws-lambda/test/test.code.ts @@ -213,6 +213,10 @@ export = { containerPath: '/src', }, ], + environment: { + VAR1: 'value1', + VAR2: 'value2', + }, command, }); @@ -220,6 +224,8 @@ export = { 'run', '--rm', '-v', `${dockerAssetPath}:/asset`, '-v', `${srcPath}:/src`, + '--env', 'VAR1=value1', + '--env', 'VAR2=value2', 'alpine', ...command, ])); From 116d9854146cafd6640392dd6b4f59259b599900 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Sat, 16 May 2020 17:47:47 +0200 Subject: [PATCH 03/53] fix integ test --- .../aws-lambda/test/integ.docker.expected.json | 18 +++++++++--------- .../@aws-cdk/aws-lambda/test/integ.docker.ts | 17 ++++++++++++++--- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json b/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json index 9816dcb18319f..577cee69b04af 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json +++ b/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json @@ -36,7 +36,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9S3BucketDED3100C" + "Ref": "AssetParameters24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4eS3Bucket36FADAC7" }, "S3Key": { "Fn::Join": [ @@ -49,7 +49,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9S3VersionKeyB85BA10F" + "Ref": "AssetParameters24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4eS3VersionKeyC112E84E" } ] } @@ -62,7 +62,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9S3VersionKeyB85BA10F" + "Ref": "AssetParameters24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4eS3VersionKeyC112E84E" } ] } @@ -87,17 +87,17 @@ } }, "Parameters": { - "AssetParameters567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9S3BucketDED3100C": { + "AssetParameters24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4eS3Bucket36FADAC7": { "Type": "String", - "Description": "S3 bucket for asset \"567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9\"" + "Description": "S3 bucket for asset \"24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4e\"" }, - "AssetParameters567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9S3VersionKeyB85BA10F": { + "AssetParameters24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4eS3VersionKeyC112E84E": { "Type": "String", - "Description": "S3 key for asset version \"567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9\"" + "Description": "S3 key for asset version \"24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4e\"" }, - "AssetParameters567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9ArtifactHash5913CC34": { + "AssetParameters24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4eArtifactHash01270413": { "Type": "String", - "Description": "Artifact hash for asset \"567c99719fe54bd90e4a5f05163782b41e552ba606dbf49929efb314d18938d9\"" + "Description": "Artifact hash for asset \"24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4e\"" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts b/packages/@aws-cdk/aws-lambda/test/integ.docker.ts index fd7ca5bbb784a..8aec286bb3f93 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.docker.ts @@ -11,9 +11,20 @@ class TestStack extends Stack { image: 'python:3.6', assetPath: path.join(__dirname, 'python-lambda-handler'), // this is /asset in the container command: [ - 'pip3', 'install', - '-r', '/asset/requirements.txt', - '-t', '/asset', + // Could normally be something like: + // ``` + // [ + // 'pip', 'install', + // '-r', '/asset/requirements.txt', + // '-t', '/asset', + // ] + // ``` + // but we need to remove the __pycache__ folders to ensure a stable + // CDK asset hash for the integ test expectation, so we do: + '/bin/bash', '-c', ` + pip install -r /asset/requirements.txt -t /asset && + find /asset -type d -name __pycache__ -exec rm -rf {} + + `, ], }), runtime: lambda.Runtime.PYTHON_3_6, From d2a66c88988ef37c3fb7741096dea01b9442d62b Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Sat, 16 May 2020 17:56:36 +0200 Subject: [PATCH 04/53] .gitignore in python-lambda-handler --- .../@aws-cdk/aws-lambda/test/python-lambda-handler/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 packages/@aws-cdk/aws-lambda/test/python-lambda-handler/.gitignore diff --git a/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/.gitignore b/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/.gitignore new file mode 100644 index 0000000000000..7e1de670e94bb --- /dev/null +++ b/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!index.py +!requirements.txt From 7120eba6bf9fe2876cbd765af0968e80b7dfd4b3 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Sat, 16 May 2020 20:52:11 +0200 Subject: [PATCH 05/53] fix integ test 2 --- packages/@aws-cdk/aws-lambda/lib/code.ts | 2 +- .../test/integ.docker.expected.json | 18 ++++----- .../@aws-cdk/aws-lambda/test/integ.docker.ts | 40 ++++++++++++------- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/code.ts b/packages/@aws-cdk/aws-lambda/lib/code.ts index 8c17ac82d246e..77c3d15935a7c 100644 --- a/packages/@aws-cdk/aws-lambda/lib/code.ts +++ b/packages/@aws-cdk/aws-lambda/lib/code.ts @@ -241,7 +241,7 @@ export interface DockerVolume { /** * Docker run options */ -export interface DockerRunOptions { +export interface DockerRunOptions extends s3_assets.AssetOptions { /** * The command to run in the container. */ diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json b/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json index 577cee69b04af..a7bce72537fe4 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json +++ b/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json @@ -36,7 +36,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4eS3Bucket36FADAC7" + "Ref": "AssetParameters0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2S3Bucket295EB1DC" }, "S3Key": { "Fn::Join": [ @@ -49,7 +49,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4eS3VersionKeyC112E84E" + "Ref": "AssetParameters0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2S3VersionKey57F2A340" } ] } @@ -62,7 +62,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4eS3VersionKeyC112E84E" + "Ref": "AssetParameters0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2S3VersionKey57F2A340" } ] } @@ -87,17 +87,17 @@ } }, "Parameters": { - "AssetParameters24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4eS3Bucket36FADAC7": { + "AssetParameters0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2S3Bucket295EB1DC": { "Type": "String", - "Description": "S3 bucket for asset \"24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4e\"" + "Description": "S3 bucket for asset \"0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2\"" }, - "AssetParameters24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4eS3VersionKeyC112E84E": { + "AssetParameters0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2S3VersionKey57F2A340": { "Type": "String", - "Description": "S3 key for asset version \"24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4e\"" + "Description": "S3 key for asset version \"0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2\"" }, - "AssetParameters24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4eArtifactHash01270413": { + "AssetParameters0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2ArtifactHash3226F4E0": { "Type": "String", - "Description": "Artifact hash for asset \"24cd7ad96e93c24ddc5059023ea542a573c013fb9c5368573d6c46fddcfdfa4e\"" + "Description": "Artifact hash for asset \"0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2\"" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts b/packages/@aws-cdk/aws-lambda/test/integ.docker.ts index 8aec286bb3f93..f6da8aca45a4b 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.docker.ts @@ -1,4 +1,6 @@ import { App, Construct, Stack, StackProps } from '@aws-cdk/core'; +import * as crypto from 'crypto'; +import * as fs from 'fs'; import * as path from 'path'; import * as lambda from '../lib'; @@ -6,26 +8,17 @@ class TestStack extends Stack { constructor(scope: Construct, id: string, props?: StackProps) { super(scope, id, props); + const assetPath = path.join(__dirname, 'python-lambda-handler'); new lambda.Function(this, 'Function', { code: lambda.Code.fromDockerImage({ image: 'python:3.6', - assetPath: path.join(__dirname, 'python-lambda-handler'), // this is /asset in the container + assetPath, // this is /asset in the container command: [ - // Could normally be something like: - // ``` - // [ - // 'pip', 'install', - // '-r', '/asset/requirements.txt', - // '-t', '/asset', - // ] - // ``` - // but we need to remove the __pycache__ folders to ensure a stable - // CDK asset hash for the integ test expectation, so we do: - '/bin/bash', '-c', ` - pip install -r /asset/requirements.txt -t /asset && - find /asset -type d -name __pycache__ -exec rm -rf {} + - `, + 'pip', 'install', + '-r', '/asset/requirements.txt', + '-t', '/asset', ], + sourceHash: calcSourceHash(assetPath), }), runtime: lambda.Runtime.PYTHON_3_6, handler: 'index.handler', @@ -36,3 +29,20 @@ class TestStack extends Stack { const app = new App(); new TestStack(app, 'cdk-integ-lambda-docker'); app.synth(); + +// Custom source hash calculation to ensure consistent behavior +// with Python dependencies. Needed for integ test expectation. +function calcSourceHash(srcDir: string): string { + const sha = crypto.createHash('sha256'); + for (const dirent of fs.readdirSync(srcDir, { withFileTypes: true })) { + if (!dirent.isFile()) { + continue; + } + const data = fs.readFileSync(path.join(srcDir, dirent.name)); + sha.update(``); + sha.update(data); + sha.update(''); + } + + return sha.digest('hex'); +} From 13057c81e28ea44c8b1319439dcc967b2aa171b7 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Sat, 16 May 2020 20:54:27 +0200 Subject: [PATCH 06/53] start of README --- packages/@aws-cdk/aws-lambda/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index 536d2a9e71d9d..555e792f329e5 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -33,6 +33,10 @@ runtime code. limited to supported runtimes and the code cannot exceed 4KiB. * `lambda.Code.fromAsset(path)` - specify a directory or a .zip file in the local filesystem which will be zipped and uploaded to S3 before deployment. + * `lambda.Code.fromDockerImage(options)` - code from a command run in an existing + Docker image. + * `lambda.Code.fromDockerAsset(options)` - code from a command run in a Docker image + built from a Dockerfile. The following example shows how to define a Python function and deploy the code from the local directory `my-lambda-handler` to it: From aad39af70c913406ca0e90fdfec85e0f9ae89724 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Sat, 16 May 2020 21:16:17 +0200 Subject: [PATCH 07/53] calcSourceHash --- packages/@aws-cdk/aws-lambda/test/integ.docker.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts b/packages/@aws-cdk/aws-lambda/test/integ.docker.ts index f6da8aca45a4b..093b0774aeac0 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.docker.ts @@ -34,12 +34,10 @@ app.synth(); // with Python dependencies. Needed for integ test expectation. function calcSourceHash(srcDir: string): string { const sha = crypto.createHash('sha256'); - for (const dirent of fs.readdirSync(srcDir, { withFileTypes: true })) { - if (!dirent.isFile()) { - continue; - } - const data = fs.readFileSync(path.join(srcDir, dirent.name)); - sha.update(``); + const files = ['index.py', 'requirements.txt']; + for (const file of files) { + const data = fs.readFileSync(path.join(srcDir, file)); + sha.update(``); sha.update(data); sha.update(''); } From 2d780e65eadccbe02d4c54efe17734bbc84f76ee Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Sat, 16 May 2020 21:35:16 +0200 Subject: [PATCH 08/53] AssetOptions --- packages/@aws-cdk/aws-lambda/lib/code.ts | 7 ++++++- .../aws-lambda/test/integ.docker.expected.json | 18 +++++++++--------- .../@aws-cdk/aws-lambda/test/integ.docker.ts | 10 ++++++---- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/code.ts b/packages/@aws-cdk/aws-lambda/lib/code.ts index 77c3d15935a7c..fb96475e8c300 100644 --- a/packages/@aws-cdk/aws-lambda/lib/code.ts +++ b/packages/@aws-cdk/aws-lambda/lib/code.ts @@ -310,7 +310,12 @@ export class DockerImageCode extends AssetCode { throw new Error(`[Status ${docker.status}] stdout: ${docker.stdout?.toString().trim()}\n\n\nstderr: ${docker.stderr?.toString().trim()}`); } - super(options.assetPath); + super(options.assetPath, { + exclude: options.exclude, + follow: options.follow, + readers: options.readers, + sourceHash: options.sourceHash, + }); } } diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json b/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json index a7bce72537fe4..c61de17cb8350 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json +++ b/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json @@ -36,7 +36,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2S3Bucket295EB1DC" + "Ref": "AssetParameters522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaaS3Bucket26FECCCE" }, "S3Key": { "Fn::Join": [ @@ -49,7 +49,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2S3VersionKey57F2A340" + "Ref": "AssetParameters522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaaS3VersionKey854ACFF5" } ] } @@ -62,7 +62,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2S3VersionKey57F2A340" + "Ref": "AssetParameters522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaaS3VersionKey854ACFF5" } ] } @@ -87,17 +87,17 @@ } }, "Parameters": { - "AssetParameters0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2S3Bucket295EB1DC": { + "AssetParameters522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaaS3Bucket26FECCCE": { "Type": "String", - "Description": "S3 bucket for asset \"0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2\"" + "Description": "S3 bucket for asset \"522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaa\"" }, - "AssetParameters0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2S3VersionKey57F2A340": { + "AssetParameters522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaaS3VersionKey854ACFF5": { "Type": "String", - "Description": "S3 key for asset version \"0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2\"" + "Description": "S3 key for asset version \"522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaa\"" }, - "AssetParameters0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2ArtifactHash3226F4E0": { + "AssetParameters522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaaArtifactHash6F4926F4": { "Type": "String", - "Description": "Artifact hash for asset \"0e37ac27267e53a7e684f1499aa955da0860cd1a9f1b29af08fac7714e4c1bd2\"" + "Description": "Artifact hash for asset \"522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaa\"" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts b/packages/@aws-cdk/aws-lambda/test/integ.docker.ts index 093b0774aeac0..f6da8aca45a4b 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.docker.ts @@ -34,10 +34,12 @@ app.synth(); // with Python dependencies. Needed for integ test expectation. function calcSourceHash(srcDir: string): string { const sha = crypto.createHash('sha256'); - const files = ['index.py', 'requirements.txt']; - for (const file of files) { - const data = fs.readFileSync(path.join(srcDir, file)); - sha.update(``); + for (const dirent of fs.readdirSync(srcDir, { withFileTypes: true })) { + if (!dirent.isFile()) { + continue; + } + const data = fs.readFileSync(path.join(srcDir, dirent.name)); + sha.update(``); sha.update(data); sha.update(''); } From 3d3e7c2047edc50901c323cbe823ece932a953e5 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 18 May 2020 12:29:49 +0200 Subject: [PATCH 09/53] PR feedback --- packages/@aws-cdk/aws-lambda/README.md | 48 +++- packages/@aws-cdk/aws-lambda/lib/code.ts | 242 +++++------------- packages/@aws-cdk/aws-lambda/lib/docker.ts | 141 ++++++++++ packages/@aws-cdk/aws-lambda/lib/index.ts | 1 + .../test/integ.docker.expected.json | 28 +- .../@aws-cdk/aws-lambda/test/integ.docker.ts | 56 ++-- .../test/python-lambda-handler/index.py | 3 + .../@aws-cdk/aws-lambda/test/test.code.ts | 193 +++++++------- 8 files changed, 408 insertions(+), 304 deletions(-) create mode 100644 packages/@aws-cdk/aws-lambda/lib/docker.ts diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index 555e792f329e5..9939462f9722d 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -32,11 +32,8 @@ runtime code. * `lambda.Code.fromInline(code)` - inline the handle code as a string. This is limited to supported runtimes and the code cannot exceed 4KiB. * `lambda.Code.fromAsset(path)` - specify a directory or a .zip file in the local - filesystem which will be zipped and uploaded to S3 before deployment. - * `lambda.Code.fromDockerImage(options)` - code from a command run in an existing - Docker image. - * `lambda.Code.fromDockerAsset(options)` - code from a command run in a Docker image - built from a Dockerfile. + filesystem which will be zipped and uploaded to S3 before deployment. See also + [using Docker with asset code](#Using-Docker-With-Asset-Code). The following example shows how to define a Python function and deploy the code from the local directory `my-lambda-handler` to it: @@ -258,6 +255,47 @@ number of times and with different properties. Using `SingletonFunction` here wi For example, the `LogRetention` construct requires only one single lambda function for all different log groups whose retention it seeks to manage. +### Using Docker with Asset Code +When using `lambda.Code.fromAsset(path)` it is possible to "act" on the code by running a +command in a Docker container. By default, the asset path is mounted in the container +at `/asset` and is set as the working directory. + +Example with Python: +```ts +new lambda.Function(this, 'Function', { + code: lambda.Code.fromAsset(path.join(__dirname, 'my-python-handler'), { + bundle: { + image: lambda.DockerImage.fromImage('python:3.6'), // Use an existing image + command: [ + 'pip', 'install', + '-r', 'requirements.txt', + '-t', '.', + ], + }, + }), + runtime: lambda.Runtime.PYTHON_3_6, + handler: 'index.handler', +}); +``` + +Use `lambda.DockerImage.fromBuild(path)` to build a specific image: + +```ts +new lambda.Function(this, 'Function', { + code: lambda.Code.fromAsset('/path/to/handler'), { + bundle: { + image: lambda.DockerImage.fromBuild('/path/to/dir/with/DockerFile', { + buildArgs: { + ARG1: 'value1', + }, + }), + command: ['my', 'cool', 'command'], + }, + }), + // ... +}); +``` + ### Language-specific APIs Language-specific higher level constructs are provided in separate modules: diff --git a/packages/@aws-cdk/aws-lambda/lib/code.ts b/packages/@aws-cdk/aws-lambda/lib/code.ts index fb96475e8c300..0e796e8c462c5 100644 --- a/packages/@aws-cdk/aws-lambda/lib/code.ts +++ b/packages/@aws-cdk/aws-lambda/lib/code.ts @@ -1,8 +1,8 @@ import * as s3 from '@aws-cdk/aws-s3'; import * as s3_assets from '@aws-cdk/aws-s3-assets'; import * as cdk from '@aws-cdk/core'; -import { spawnSync } from 'child_process'; import * as fs from 'fs'; +import { DockerImage, DockerVolume } from './docker'; export abstract class Code { /** @@ -42,24 +42,10 @@ export abstract class Code { * * @param path Either a directory with the Lambda code bundle or a .zip file */ - public static fromAsset(path: string, options?: s3_assets.AssetOptions): AssetCode { + public static fromAsset(path: string, options?: AssetCodeOptions): AssetCode { return new AssetCode(path, options); } - /** - * Lambda code from a command run in an existing Docker image. - */ - public static fromDockerImage(options: DockerImageCodeOptions): AssetCode { - return new DockerImageCode(options); - } - - /** - * Lambda code from a command run in a Docker image built from a Dockerfile. - */ - public static fromDockerAsset(options: DockerAssetCodeOptions): AssetCode { - return new DockerAssetCode(options); - } - /** * @deprecated use `fromAsset` */ @@ -177,200 +163,118 @@ export class InlineCode extends Code { } /** - * Lambda code from a local directory. + * Bundle options */ -export class AssetCode extends Code { - public readonly isInline = false; - private asset?: s3_assets.Asset; - +export interface BundleOptions { /** - * @param path The path to the asset file or directory. - */ - constructor(public readonly path: string, private readonly options: s3_assets.AssetOptions = { }) { - super(); - } - - public bind(scope: cdk.Construct): CodeConfig { - // If the same AssetCode is used multiple times, retain only the first instantiation. - if (!this.asset) { - this.asset = new s3_assets.Asset(scope, 'Code', { - path: this.path, - ...this.options, - }); - } - - if (!this.asset.isZipArchive) { - throw new Error(`Asset must be a .zip file or a directory (${this.path})`); - } - - return { - s3Location: { - bucketName: this.asset.s3BucketName, - objectKey: this.asset.s3ObjectKey, - }, - }; - } - - public bindToResource(resource: cdk.CfnResource, options: ResourceBindOptions = { }) { - if (!this.asset) { - throw new Error('bindToResource() must be called after bind()'); - } - - const resourceProperty = options.resourceProperty || 'Code'; - - // https://github.com/aws/aws-cdk/issues/1432 - this.asset.addResourceMetadata(resource, resourceProperty); - } -} - -/** - * A Docker volume - */ -export interface DockerVolume { - /** - * The path to the file or directory on the host machine + * The Docker image where the command will run. */ - readonly hostPath: string; + readonly image: DockerImage; - /** - * The path where the file or directory is mounted in the container - */ - readonly containerPath: string; -} - -/** - * Docker run options - */ -export interface DockerRunOptions extends s3_assets.AssetOptions { /** * The command to run in the container. */ readonly command: string[]; /** - * The path of the asset directory that will contain the build output of the Docker - * container. This path is mounted at `/asset` in the container. It is created - * if it doesn't exist. - */ - readonly assetPath: string; - - /** - * Additional Docker volumes to mount. + * Docker volumes to mount. * - * @default - no additional volumes are mounted + * @default - The path to the asset file or directory is mounted at /asset */ readonly volumes?: DockerVolume[]; /** * The environment variables to pass to the container. * - * @default - No environment variables. + * @default - no environment variables. */ readonly environment?: { [key: string]: string; }; -} -/** - * Options for DockerImageCode - */ -export interface DockerImageCodeOptions extends DockerRunOptions { /** - * The Docker image where the command will run. + * Working directory inside the container. + * + * @default - the `containerPath` of the first mounted volume. */ - readonly image: string; + readonly workingDirectory?: string; } /** - * Lambda code from a command run in an existing Docker image + * Asset code options */ -export class DockerImageCode extends AssetCode { - constructor(options: DockerImageCodeOptions) { - if (!fs.existsSync(options.assetPath)) { - fs.mkdirSync(options.assetPath); - } - - const volumes = options.volumes || []; - const environment = options.environment || {}; - - const dockerArgs: string[] = [ - 'run', '--rm', - '-v', `${options.assetPath}:/asset`, - ...flatten(volumes.map(v => ['-v', `${v.hostPath}:${v.containerPath}`])), - ...flatten(Object.entries(environment).map(([k, v]) => ['--env', `${k}=${v}`])), - options.image, - ...options.command, - ]; - - const docker = spawnSync('docker', dockerArgs); - - if (docker.error) { - throw docker.error; - } - - if (docker.status !== 0) { - throw new Error(`[Status ${docker.status}] stdout: ${docker.stdout?.toString().trim()}\n\n\nstderr: ${docker.stderr?.toString().trim()}`); - } - - super(options.assetPath, { - exclude: options.exclude, - follow: options.follow, - readers: options.readers, - sourceHash: options.sourceHash, - }); - } -} - -/** - * Options for DockerAssetCode - */ -export interface DockerAssetCodeOptions extends DockerRunOptions { +export interface AssetCodeOptions extends s3_assets.AssetOptions { /** - * The path to the directory containing the Docker file. - */ - readonly dockerPath: string; - - /** - * Build args + * Bundle options * - * @default - no build args + * @default - no bundling */ - readonly buildArgs?: { [key: string]: string }; + readonly bundle?: BundleOptions; } /** - * Lambda code from a command run in a Docker image built from a Dockerfile. + * Lambda code from a local directory. */ -export class DockerAssetCode extends DockerImageCode { - constructor(options: DockerAssetCodeOptions) { - const buildArgs = options.buildArgs || {}; +export class AssetCode extends Code { + public readonly isInline = false; + private asset?: s3_assets.Asset; - const dockerArgs: string[] = [ - 'build', - ...flatten(Object.entries(buildArgs).map(([k, v]) => ['--build-arg', `${k}=${v}`])), - options.dockerPath, - ]; + /** + * @param path The path to the asset file or directory. + */ + constructor(public readonly path: string, private readonly options: AssetCodeOptions = {}) { + super(); + } - const docker = spawnSync('docker', dockerArgs); + public bind(scope: cdk.Construct): CodeConfig { + if (this.options.bundle) { + // We are going to mount it, so ensure it exists + if (!fs.existsSync(this.path)) { + fs.mkdirSync(this.path); + } + + const volumes = this.options.bundle.volumes ?? [ + { + hostPath: this.path, + containerPath: '/asset', + }, + ]; + + this.options.bundle.image.run({ + command: this.options.bundle.command, + volumes, + environment: this.options.bundle.environment, + workingDirectory: this.options.bundle.workingDirectory ?? volumes[0].containerPath, + }); + } - if (docker.error) { - throw docker.error; + // If the same AssetCode is used multiple times, retain only the first instantiation. + if (!this.asset) { + this.asset = new s3_assets.Asset(scope, 'Code', { + path: this.path, + ...this.options, + }); } - if (docker.status !== 0) { - throw new Error(`[Status ${docker.status}] stdout: ${docker.stdout?.toString().trim()}\n\n\nstderr: ${docker.stderr?.toString().trim()}`); + if (!this.asset.isZipArchive) { + throw new Error(`Asset must be a .zip file or a directory (${this.path})`); } - const match = docker.stdout.toString().match(/Successfully built ([a-z0-9]+)/); + return { + s3Location: { + bucketName: this.asset.s3BucketName, + objectKey: this.asset.s3ObjectKey, + }, + }; + } - if (!match) { - throw new Error('Failed to extract image ID from Docker build output'); + public bindToResource(resource: cdk.CfnResource, options: ResourceBindOptions = { }) { + if (!this.asset) { + throw new Error('bindToResource() must be called after bind()'); } - super({ - assetPath: options.assetPath, - command: options.command, - image: match[1], - volumes: options.volumes, - }); + const resourceProperty = options.resourceProperty || 'Code'; + + // https://github.com/aws/aws-cdk/issues/1432 + this.asset.addResourceMetadata(resource, resourceProperty); } } @@ -480,7 +384,3 @@ export class CfnParametersCode extends Code { } } } - -function flatten(x: string[][]) { - return Array.prototype.concat([], ...x); -} diff --git a/packages/@aws-cdk/aws-lambda/lib/docker.ts b/packages/@aws-cdk/aws-lambda/lib/docker.ts new file mode 100644 index 0000000000000..98a66d7f0f8ef --- /dev/null +++ b/packages/@aws-cdk/aws-lambda/lib/docker.ts @@ -0,0 +1,141 @@ +import { spawnSync } from 'child_process'; + +/** + * A Docker volume + */ +export interface DockerVolume { + /** + * The path to the file or directory on the host machine + */ + readonly hostPath: string; + + /** + * The path where the file or directory is mounted in the container + */ + readonly containerPath: string; +} + +/** + * Docker run options + */ +export interface DockerRunOptions { + /** + * The command to run in the container. + */ + readonly command: string[]; + + /** + * Docker volumes to mount. + * + * @default - no volumes are mounted + */ + readonly volumes?: DockerVolume[]; + + /** + * The environment variables to pass to the container. + * + * @default - no environment variables. + */ + readonly environment?: { [key: string]: string; }; + + /** + * Working directory inside the container. + * + * @default - image default + */ + readonly workingDirectory?: string; +} + +/** + * Docker build options + */ +export interface DockerBuildOptions { + /** + * Build args + * + * @default - no build args + */ + readonly buildArgs?: { [key: string]: string }; +} + +/** + * A Docker image + */ +export class DockerImage { + /** + * Use an existing Docker image + * + * @param image the image name + */ + public static fromImage(image: string) { + return new DockerImage(image); + } + + /** + * Build a Docker image + * + * @param path The path to the directory containing the Docker file + * @param options Docker build options + */ + public static fromBuild(path: string, options: DockerBuildOptions = {}) { + const buildArgs = options.buildArgs || {}; + + const dockerArgs: string[] = [ + 'build', + ...flatten(Object.entries(buildArgs).map(([k, v]) => ['--build-arg', `${k}=${v}`])), + path, + ]; + + const docker = exec('docker', dockerArgs); + + const match = docker.stdout.toString().match(/Successfully built ([a-z0-9]+)/); + + if (!match) { + throw new Error('Failed to extract image ID from Docker build output'); + } + + return new DockerImage(match[1]); + } + + /** @param image The Docker image */ + constructor(public readonly image: string) {} + + /** + * Runs a Docker image + */ + public run(options: DockerRunOptions) { + const volumes = options.volumes || []; + const environment = options.environment || {}; + + const dockerArgs: string[] = [ + 'run', '--rm', + ...flatten(volumes.map(v => ['-v', `${v.hostPath}:${v.containerPath}`])), + ...flatten(Object.entries(environment).map(([k, v]) => ['--env', `${k}=${v}`])), + ...options.workingDirectory + ? ['-w', options.workingDirectory] + : [], + this.image, + ...options.command, + ]; + + exec('docker', dockerArgs); + } +} + +function flatten(x: string[][]) { + return Array.prototype.concat([], ...x); +} + +function exec(cmd: string, args: string[]) { + const proc = spawnSync(cmd, args); + + if (proc.error) { + throw proc.error; + } + + if (proc.status !== 0) { + throw new Error(`[Status ${proc.status}] stdout: ${proc.stdout?.toString().trim()}\n\n\nstderr: ${proc.stderr?.toString().trim()}`); + } + + return proc; +} diff --git a/packages/@aws-cdk/aws-lambda/lib/index.ts b/packages/@aws-cdk/aws-lambda/lib/index.ts index b494e924c604a..76fcaee327896 100644 --- a/packages/@aws-cdk/aws-lambda/lib/index.ts +++ b/packages/@aws-cdk/aws-lambda/lib/index.ts @@ -12,6 +12,7 @@ export * from './event-source'; export * from './event-source-mapping'; export * from './destination'; export * from './event-invoke-config'; +export * from './docker'; export * from './log-retention'; diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json b/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json index c61de17cb8350..ecc3743fe645e 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json +++ b/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json @@ -36,7 +36,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaaS3Bucket26FECCCE" + "Ref": "AssetParameters32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9S3BucketCB34B9C8" }, "S3Key": { "Fn::Join": [ @@ -49,7 +49,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaaS3VersionKey854ACFF5" + "Ref": "AssetParameters32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9S3VersionKey037CDB93" } ] } @@ -62,7 +62,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaaS3VersionKey854ACFF5" + "Ref": "AssetParameters32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9S3VersionKey037CDB93" } ] } @@ -87,17 +87,27 @@ } }, "Parameters": { - "AssetParameters522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaaS3Bucket26FECCCE": { + "AssetParameters32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9S3BucketCB34B9C8": { "Type": "String", - "Description": "S3 bucket for asset \"522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaa\"" + "Description": "S3 bucket for asset \"32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9\"" }, - "AssetParameters522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaaS3VersionKey854ACFF5": { + "AssetParameters32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9S3VersionKey037CDB93": { "Type": "String", - "Description": "S3 key for asset version \"522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaa\"" + "Description": "S3 key for asset version \"32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9\"" }, - "AssetParameters522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaaArtifactHash6F4926F4": { + "AssetParameters32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9ArtifactHash24D3E285": { "Type": "String", - "Description": "Artifact hash for asset \"522c42c95cdf43e7a7d9ba7a91ea9a34fa998acb8ee1985cd843fb453f276eaa\"" + "Description": "Artifact hash for asset \"32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9\"" + } + }, + "Outputs": { + "FunctionArn": { + "Value": { + "Fn::GetAtt": [ + "Function76856677", + "Arn" + ] + } } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts b/packages/@aws-cdk/aws-lambda/test/integ.docker.ts index f6da8aca45a4b..ae0cffd19b498 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.docker.ts @@ -1,48 +1,44 @@ -import { App, Construct, Stack, StackProps } from '@aws-cdk/core'; -import * as crypto from 'crypto'; -import * as fs from 'fs'; +import { App, CfnOutput, Construct, FileSystem, Stack, StackProps } from '@aws-cdk/core'; import * as path from 'path'; import * as lambda from '../lib'; +/** + * Stack verification steps: + * * aws cloudformation describe-stacks --stack-name cdk-integ-lambda-docker --query Stacks[0].Outputs[0].OutputValue + * * aws lambda invoke --function-name response.json + * * cat response.json + * The last command should show '200' + */ class TestStack extends Stack { constructor(scope: Construct, id: string, props?: StackProps) { super(scope, id, props); const assetPath = path.join(__dirname, 'python-lambda-handler'); - new lambda.Function(this, 'Function', { - code: lambda.Code.fromDockerImage({ - image: 'python:3.6', - assetPath, // this is /asset in the container - command: [ - 'pip', 'install', - '-r', '/asset/requirements.txt', - '-t', '/asset', - ], - sourceHash: calcSourceHash(assetPath), + const fn = new lambda.Function(this, 'Function', { + code: lambda.Code.fromAsset(assetPath, { + bundle: { + image: lambda.DockerImage.fromImage('python:3.6'), + command: [ + 'pip', 'install', + '-r', 'requirements.txt', + '-t', '.', + ], + }, + // Python dependencies do not give a stable hash + sourceHash: FileSystem.fingerprint(assetPath, { + exclude: ['*', '!index.py', '!requirements.txt'], + }), }), runtime: lambda.Runtime.PYTHON_3_6, handler: 'index.handler', }); + + new CfnOutput(this, 'FunctionArn', { + value: fn.functionArn, + }); } } const app = new App(); new TestStack(app, 'cdk-integ-lambda-docker'); app.synth(); - -// Custom source hash calculation to ensure consistent behavior -// with Python dependencies. Needed for integ test expectation. -function calcSourceHash(srcDir: string): string { - const sha = crypto.createHash('sha256'); - for (const dirent of fs.readdirSync(srcDir, { withFileTypes: true })) { - if (!dirent.isFile()) { - continue; - } - const data = fs.readFileSync(path.join(srcDir, dirent.name)); - sha.update(``); - sha.update(data); - sha.update(''); - } - - return sha.digest('hex'); -} diff --git a/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/index.py b/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/index.py index 960064a9d5b4e..175a36616590a 100644 --- a/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/index.py +++ b/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/index.py @@ -2,4 +2,7 @@ def handler(event, context): r = requests.get('https://aws.amazon.com') + print(r.status_code) + + return r.status_code diff --git a/packages/@aws-cdk/aws-lambda/test/test.code.ts b/packages/@aws-cdk/aws-lambda/test/test.code.ts index 9d797e44b0cc9..0fe6b3f70abbc 100644 --- a/packages/@aws-cdk/aws-lambda/test/test.code.ts +++ b/packages/@aws-cdk/aws-lambda/test/test.code.ts @@ -84,6 +84,110 @@ export = { }, ResourcePart.CompleteDefinition)); test.done(); }, + + 'docker image'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + + const dockerAssetPath = 'asset-path'; + const command = ['this', 'is', 'a', 'build', 'command']; + + // WHEN + new lambda.Function(stack, 'Fn', { + handler: 'foom', + runtime: lambda.Runtime.NODEJS_12_X, + code: lambda.Code.fromAsset(dockerAssetPath, { + bundle: { + image: lambda.DockerImage.fromImage('alpine'), + environment: { + VAR1: 'value1', + VAR2: 'value2', + }, + command, + }, + }), + }); + + // THEN + test.ok(spawnSyncStub.calledWith('docker', [ + 'run', '--rm', + '-v', `${dockerAssetPath}:/asset`, + '--env', 'VAR1=value1', + '--env', 'VAR2=value2', + '-w', '/asset', + 'alpine', + ...command, + ]), 'docker run not called with expected args'); + + spawnSyncStub.restore(); + + test.done(); + }, + + 'docker build'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + + const imageId = 'abcdef123456'; + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from(`Successfully built ${imageId}`), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + + const dockerPath = 'docker-path'; + const testArg = 'cdk-test'; + const dockerAssetPath = 'asset-path'; + const command = ['this', 'is', 'a', 'build', 'command']; + + // WHEN + new lambda.Function(stack, 'Fn', { + handler: 'foom', + runtime: lambda.Runtime.NODEJS_12_X, + code: lambda.Code.fromAsset(dockerAssetPath, { + bundle: { + image: lambda.DockerImage.fromBuild(dockerPath, { + buildArgs: { + TEST_ARG: testArg, + }, + }), + command, + }, + }), + }); + + // THEN + test.ok(spawnSyncStub.calledWith('docker', [ + 'build', + '--build-arg', `TEST_ARG=${testArg}`, + dockerPath, + ]), 'docker build not called with expected args'); + + test.ok(spawnSyncStub.calledWith('docker', [ + 'run', '--rm', + '-v', `${dockerAssetPath}:/asset`, + '-w', '/asset', + imageId, + ...command, + ]), 'docker run not called with expected args'); + + spawnSyncStub.restore(); + + test.done(); + + }, }, 'lambda.Code.fromCfnParameters': { @@ -189,95 +293,6 @@ export = { test.done(); }, }, - - 'lambda.Code.fromDockerImage'(test: Test) { - const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - }); - - const dockerAssetPath = 'asset-path'; - const srcPath = 'src-path'; - const command = ['this', 'is', 'a', 'build', 'command']; - - lambda.Code.fromDockerImage({ - assetPath: dockerAssetPath, - image: 'alpine', - volumes: [ - { - hostPath: srcPath, - containerPath: '/src', - }, - ], - environment: { - VAR1: 'value1', - VAR2: 'value2', - }, - command, - }); - - test.ok(spawnSyncStub.calledWith('docker', [ - 'run', '--rm', - '-v', `${dockerAssetPath}:/asset`, - '-v', `${srcPath}:/src`, - '--env', 'VAR1=value1', - '--env', 'VAR2=value2', - 'alpine', - ...command, - ])); - - spawnSyncStub.restore(); - - test.done(); - }, - - 'lambda.Code.fromDockerAsset'(test: Test) { - const imageId = 'abcdef123456'; - const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from(`Successfully built ${imageId}`), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - }); - - const dockerPath = 'docker-path'; - const testArg = 'cdk-test'; - const dockerAssetPath = 'asset-path'; - const command = ['this', 'is', 'a', 'build', 'command']; - - lambda.Code.fromDockerAsset({ - dockerPath, - buildArgs: { - TEST_ARG: testArg, - }, - assetPath: dockerAssetPath, - command, - }); - - test.ok(spawnSyncStub.calledWith('docker', [ - 'build', - '--build-arg', `TEST_ARG=${testArg}`, - dockerPath, - ])); - - test.ok(spawnSyncStub.calledWith('docker', [ - 'run', '--rm', - '-v', `${dockerAssetPath}:/asset`, - imageId, - ...command, - ])); - - spawnSyncStub.restore(); - - test.done(); - - }, }; function defineFunction(code: lambda.Code, runtime: lambda.Runtime = lambda.Runtime.NODEJS_10_X) { From 4f68287dd717a2ff2d98ff609f2e0d356c39daba Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 18 May 2020 16:16:38 +0200 Subject: [PATCH 10/53] docker -> bundling --- packages/@aws-cdk/aws-lambda/README.md | 8 +++--- .../aws-lambda/lib/{docker.ts => bundling.ts} | 16 +++++------ packages/@aws-cdk/aws-lambda/lib/code.ts | 28 ++++++++++--------- packages/@aws-cdk/aws-lambda/lib/index.ts | 2 +- ...cted.json => integ.bundling.expected.json} | 0 .../{integ.docker.ts => integ.bundling.ts} | 4 +-- .../@aws-cdk/aws-lambda/test/test.code.ts | 8 +++--- 7 files changed, 34 insertions(+), 32 deletions(-) rename packages/@aws-cdk/aws-lambda/lib/{docker.ts => bundling.ts} (87%) rename packages/@aws-cdk/aws-lambda/test/{integ.docker.expected.json => integ.bundling.expected.json} (100%) rename packages/@aws-cdk/aws-lambda/test/{integ.docker.ts => integ.bundling.ts} (93%) diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index 9939462f9722d..be2e8804e9326 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -33,7 +33,7 @@ runtime code. limited to supported runtimes and the code cannot exceed 4KiB. * `lambda.Code.fromAsset(path)` - specify a directory or a .zip file in the local filesystem which will be zipped and uploaded to S3 before deployment. See also - [using Docker with asset code](#Using-Docker-With-Asset-Code). + [bundling asset code](#Bundling-Asset-Code). The following example shows how to define a Python function and deploy the code from the local directory `my-lambda-handler` to it: @@ -255,7 +255,7 @@ number of times and with different properties. Using `SingletonFunction` here wi For example, the `LogRetention` construct requires only one single lambda function for all different log groups whose retention it seeks to manage. -### Using Docker with Asset Code +### Bundling Asset Code When using `lambda.Code.fromAsset(path)` it is possible to "act" on the code by running a command in a Docker container. By default, the asset path is mounted in the container at `/asset` and is set as the working directory. @@ -265,7 +265,7 @@ Example with Python: new lambda.Function(this, 'Function', { code: lambda.Code.fromAsset(path.join(__dirname, 'my-python-handler'), { bundle: { - image: lambda.DockerImage.fromImage('python:3.6'), // Use an existing image + image: lambda.BundlingDockerImage.fromRegistry('python:3.6'), // Use an existing image command: [ 'pip', 'install', '-r', 'requirements.txt', @@ -284,7 +284,7 @@ Use `lambda.DockerImage.fromBuild(path)` to build a specific image: new lambda.Function(this, 'Function', { code: lambda.Code.fromAsset('/path/to/handler'), { bundle: { - image: lambda.DockerImage.fromBuild('/path/to/dir/with/DockerFile', { + image: lambda.BundlingDockerImage.fromAsset('/path/to/dir/with/DockerFile', { buildArgs: { ARG1: 'value1', }, diff --git a/packages/@aws-cdk/aws-lambda/lib/docker.ts b/packages/@aws-cdk/aws-lambda/lib/bundling.ts similarity index 87% rename from packages/@aws-cdk/aws-lambda/lib/docker.ts rename to packages/@aws-cdk/aws-lambda/lib/bundling.ts index 98a66d7f0f8ef..57e90ad889634 100644 --- a/packages/@aws-cdk/aws-lambda/lib/docker.ts +++ b/packages/@aws-cdk/aws-lambda/lib/bundling.ts @@ -59,25 +59,25 @@ export interface DockerBuildOptions { } /** - * A Docker image + * A Docker image used for Lambda bundling */ -export class DockerImage { +export class BundlingDockerImage { /** - * Use an existing Docker image + * Reference an image on DockerHub or another online registry. * * @param image the image name */ - public static fromImage(image: string) { - return new DockerImage(image); + public static fromRegistry(image: string) { + return new BundlingDockerImage(image); } /** - * Build a Docker image + * Reference an image that's built directly from sources on disk. * * @param path The path to the directory containing the Docker file * @param options Docker build options */ - public static fromBuild(path: string, options: DockerBuildOptions = {}) { + public static fromAsset(path: string, options: DockerBuildOptions = {}) { const buildArgs = options.buildArgs || {}; const dockerArgs: string[] = [ @@ -94,7 +94,7 @@ export class DockerImage { throw new Error('Failed to extract image ID from Docker build output'); } - return new DockerImage(match[1]); + return new BundlingDockerImage(match[1]); } /** @param image The Docker image */ diff --git a/packages/@aws-cdk/aws-lambda/lib/code.ts b/packages/@aws-cdk/aws-lambda/lib/code.ts index 0e796e8c462c5..be7a346d004d0 100644 --- a/packages/@aws-cdk/aws-lambda/lib/code.ts +++ b/packages/@aws-cdk/aws-lambda/lib/code.ts @@ -2,7 +2,7 @@ import * as s3 from '@aws-cdk/aws-s3'; import * as s3_assets from '@aws-cdk/aws-s3-assets'; import * as cdk from '@aws-cdk/core'; import * as fs from 'fs'; -import { DockerImage, DockerVolume } from './docker'; +import { BundlingDockerImage, DockerVolume } from './bundling'; export abstract class Code { /** @@ -163,16 +163,18 @@ export class InlineCode extends Code { } /** - * Bundle options + * Bundling options */ -export interface BundleOptions { +export interface BundlingOptions { /** * The Docker image where the command will run. */ - readonly image: DockerImage; + readonly image: BundlingDockerImage; /** * The command to run in the container. + * + * @example ['npm', 'install'] */ readonly command: string[]; @@ -203,11 +205,11 @@ export interface BundleOptions { */ export interface AssetCodeOptions extends s3_assets.AssetOptions { /** - * Bundle options + * Bundle the Lambda code by executing a command in a Docker container. * - * @default - no bundling + * @default - asset path is zipped as is */ - readonly bundle?: BundleOptions; + readonly bundling?: BundlingOptions; } /** @@ -225,24 +227,24 @@ export class AssetCode extends Code { } public bind(scope: cdk.Construct): CodeConfig { - if (this.options.bundle) { + if (this.options.bundling) { // We are going to mount it, so ensure it exists if (!fs.existsSync(this.path)) { fs.mkdirSync(this.path); } - const volumes = this.options.bundle.volumes ?? [ + const volumes = this.options.bundling.volumes ?? [ { hostPath: this.path, containerPath: '/asset', }, ]; - this.options.bundle.image.run({ - command: this.options.bundle.command, + this.options.bundling.image.run({ + command: this.options.bundling.command, volumes, - environment: this.options.bundle.environment, - workingDirectory: this.options.bundle.workingDirectory ?? volumes[0].containerPath, + environment: this.options.bundling.environment, + workingDirectory: this.options.bundling.workingDirectory ?? volumes[0].containerPath, }); } diff --git a/packages/@aws-cdk/aws-lambda/lib/index.ts b/packages/@aws-cdk/aws-lambda/lib/index.ts index 76fcaee327896..e77a1880d808a 100644 --- a/packages/@aws-cdk/aws-lambda/lib/index.ts +++ b/packages/@aws-cdk/aws-lambda/lib/index.ts @@ -12,7 +12,7 @@ export * from './event-source'; export * from './event-source-mapping'; export * from './destination'; export * from './event-invoke-config'; -export * from './docker'; +export * from './bundling'; export * from './log-retention'; diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json b/packages/@aws-cdk/aws-lambda/test/integ.bundling.expected.json similarity index 100% rename from packages/@aws-cdk/aws-lambda/test/integ.docker.expected.json rename to packages/@aws-cdk/aws-lambda/test/integ.bundling.expected.json diff --git a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts similarity index 93% rename from packages/@aws-cdk/aws-lambda/test/integ.docker.ts rename to packages/@aws-cdk/aws-lambda/test/integ.bundling.ts index ae0cffd19b498..74df35f3d092c 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.docker.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts @@ -16,8 +16,8 @@ class TestStack extends Stack { const assetPath = path.join(__dirname, 'python-lambda-handler'); const fn = new lambda.Function(this, 'Function', { code: lambda.Code.fromAsset(assetPath, { - bundle: { - image: lambda.DockerImage.fromImage('python:3.6'), + bundling: { + image: lambda.BundlingDockerImage.fromRegistry('python:3.6'), command: [ 'pip', 'install', '-r', 'requirements.txt', diff --git a/packages/@aws-cdk/aws-lambda/test/test.code.ts b/packages/@aws-cdk/aws-lambda/test/test.code.ts index 0fe6b3f70abbc..c99c6b0f400e9 100644 --- a/packages/@aws-cdk/aws-lambda/test/test.code.ts +++ b/packages/@aws-cdk/aws-lambda/test/test.code.ts @@ -106,8 +106,8 @@ export = { handler: 'foom', runtime: lambda.Runtime.NODEJS_12_X, code: lambda.Code.fromAsset(dockerAssetPath, { - bundle: { - image: lambda.DockerImage.fromImage('alpine'), + bundling: { + image: lambda.BundlingDockerImage.fromRegistry('alpine'), environment: { VAR1: 'value1', VAR2: 'value2', @@ -157,8 +157,8 @@ export = { handler: 'foom', runtime: lambda.Runtime.NODEJS_12_X, code: lambda.Code.fromAsset(dockerAssetPath, { - bundle: { - image: lambda.DockerImage.fromBuild(dockerPath, { + bundling: { + image: lambda.BundlingDockerImage.fromAsset(dockerPath, { buildArgs: { TEST_ARG: testArg, }, From 8999ded3c39a4a57dfdacb7a7758a112ad20bf29 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 18 May 2020 16:44:14 +0200 Subject: [PATCH 11/53] default to lambdaci/lambda --- packages/@aws-cdk/aws-lambda/lib/code.ts | 19 ++++++++- packages/@aws-cdk/aws-lambda/lib/function.ts | 7 ++-- .../@aws-cdk/aws-lambda/test/test.code.ts | 39 +++++++++++++++++++ 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/code.ts b/packages/@aws-cdk/aws-lambda/lib/code.ts index be7a346d004d0..2c776433dda8c 100644 --- a/packages/@aws-cdk/aws-lambda/lib/code.ts +++ b/packages/@aws-cdk/aws-lambda/lib/code.ts @@ -3,6 +3,7 @@ import * as s3_assets from '@aws-cdk/aws-s3-assets'; import * as cdk from '@aws-cdk/core'; import * as fs from 'fs'; import { BundlingDockerImage, DockerVolume } from './bundling'; +import { Function } from './function'; export abstract class Code { /** @@ -168,8 +169,11 @@ export class InlineCode extends Code { export interface BundlingOptions { /** * The Docker image where the command will run. + * + * @default - The lambci/lambda build image for the function's runtime + * https://hub.docker.com/r/lambci/lambda/ */ - readonly image: BundlingDockerImage; + readonly image?: BundlingDockerImage; /** * The command to run in the container. @@ -240,7 +244,18 @@ export class AssetCode extends Code { }, ]; - this.options.bundling.image.run({ + let image: BundlingDockerImage; + if (this.options.bundling.image) { + image = this.options.bundling.image; + } else { + if (scope instanceof Function) { + image = BundlingDockerImage.fromRegistry(`lambci/lambda:build-${scope.runtime.name}`); + } else { + throw new Error('Cannot derive default bundling image from scope. Please specify an image.'); + } + } + + image.run({ command: this.options.bundling.command, volumes, environment: this.options.bundling.environment, diff --git a/packages/@aws-cdk/aws-lambda/lib/function.ts b/packages/@aws-cdk/aws-lambda/lib/function.ts index f66a67f07e336..8b4d20a4bc84c 100644 --- a/packages/@aws-cdk/aws-lambda/lib/function.ts +++ b/packages/@aws-cdk/aws-lambda/lib/function.ts @@ -485,8 +485,9 @@ export class Function extends FunctionBase { this.role.addToPolicy(statement); } + this.runtime = props.runtime; const code = props.code.bind(this); - verifyCodeConfig(code, props.runtime); + verifyCodeConfig(code, this.runtime); this.deadLetterQueue = this.buildDeadLetterQueue(props); @@ -522,8 +523,6 @@ export class Function extends FunctionBase { sep: ':', }); - this.runtime = props.runtime; - if (props.layers) { this.addLayers(...props.layers); } @@ -826,4 +825,4 @@ export function verifyCodeConfig(code: CodeConfig, runtime: Runtime) { if (code.inlineCode && !runtime.supportsInlineCode) { throw new Error(`Inline source not allowed for ${runtime.name}`); } -} \ No newline at end of file +} diff --git a/packages/@aws-cdk/aws-lambda/test/test.code.ts b/packages/@aws-cdk/aws-lambda/test/test.code.ts index c99c6b0f400e9..8ec5a7ad6a0dc 100644 --- a/packages/@aws-cdk/aws-lambda/test/test.code.ts +++ b/packages/@aws-cdk/aws-lambda/test/test.code.ts @@ -133,6 +133,45 @@ export = { test.done(); }, + 'docker image default to lambci'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + + const dockerAssetPath = 'asset-path'; + const command = ['this', 'is', 'a', 'build', 'command']; + + // WHEN + new lambda.Function(stack, 'Fn', { + handler: 'foom', + runtime: lambda.Runtime.NODEJS_12_X, + code: lambda.Code.fromAsset(dockerAssetPath, { + bundling: { command }, + }), + }); + + // THEN + test.ok(spawnSyncStub.calledWith('docker', [ + 'run', '--rm', + '-v', `${dockerAssetPath}:/asset`, + '-w', '/asset', + 'lambci/lambda:build-nodejs12.x', + ...command, + ]), 'docker run not called with expected args'); + + spawnSyncStub.restore(); + + test.done(); + }, + 'docker build'(test: Test) { // GIVEN const stack = new cdk.Stack(); From d98bc3c9a710cab142ca1ed1ca4f49f5793d442a Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 18 May 2020 17:43:35 +0200 Subject: [PATCH 12/53] /src and /bundle --- packages/@aws-cdk/aws-lambda/.gitignore | 2 + packages/@aws-cdk/aws-lambda/README.md | 24 ++++++---- packages/@aws-cdk/aws-lambda/lib/code.ts | 46 +++++++++++++++---- .../test/integ.bundling.expected.json | 18 ++++---- .../aws-lambda/test/integ.bundling.ts | 12 ++--- .../test/python-lambda-handler/.gitignore | 4 -- .../@aws-cdk/aws-lambda/test/test.code.ts | 15 +++--- 7 files changed, 75 insertions(+), 46 deletions(-) delete mode 100644 packages/@aws-cdk/aws-lambda/test/python-lambda-handler/.gitignore diff --git a/packages/@aws-cdk/aws-lambda/.gitignore b/packages/@aws-cdk/aws-lambda/.gitignore index 2d2f100c9395d..7b043ef8be063 100644 --- a/packages/@aws-cdk/aws-lambda/.gitignore +++ b/packages/@aws-cdk/aws-lambda/.gitignore @@ -15,3 +15,5 @@ nyc.config.js .LAST_PACKAGE *.snk !.eslintrc.js + +.bundle diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index be2e8804e9326..531a5d432ad86 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -256,20 +256,21 @@ For example, the `LogRetention` construct requires only one single lambda functi retention it seeks to manage. ### Bundling Asset Code -When using `lambda.Code.fromAsset(path)` it is possible to "act" on the code by running a -command in a Docker container. By default, the asset path is mounted in the container -at `/asset` and is set as the working directory. +When using `lambda.Code.fromAsset(path)` it is possible to bundle the code by running a +command in a Docker container. The asset path will be mounted at `/src`. The Docker +container is responsible for putting content at `/bundle`. The content at `/bundle` +will be zipped and used as Lambda code. Example with Python: ```ts new lambda.Function(this, 'Function', { code: lambda.Code.fromAsset(path.join(__dirname, 'my-python-handler'), { - bundle: { - image: lambda.BundlingDockerImage.fromRegistry('python:3.6'), // Use an existing image + bundling: { // Docker image defaults to the lambci/lambda build image for the function's runtime command: [ - 'pip', 'install', - '-r', 'requirements.txt', - '-t', '.', + 'bash', '-c', ` + pip install -r /src/requirements.txt -t /bundle && + rsync -r /src/ /bundle + `, ], }, }), @@ -277,13 +278,16 @@ new lambda.Function(this, 'Function', { handler: 'index.handler', }); ``` +The Docker image defaults to the [lambci/lambda](https://hub.docker.com/r/lambci/lambda/) build +image for the function's runtime. -Use `lambda.DockerImage.fromBuild(path)` to build a specific image: +Use `lambda.DockerImage.fromRegistry(image)` to use an existing image or +`lambda.DockerImage.fromAsset(path)` to build a specific image: ```ts new lambda.Function(this, 'Function', { code: lambda.Code.fromAsset('/path/to/handler'), { - bundle: { + bundling: { image: lambda.BundlingDockerImage.fromAsset('/path/to/dir/with/DockerFile', { buildArgs: { ARG1: 'value1', diff --git a/packages/@aws-cdk/aws-lambda/lib/code.ts b/packages/@aws-cdk/aws-lambda/lib/code.ts index 2c776433dda8c..3b50dc528d77f 100644 --- a/packages/@aws-cdk/aws-lambda/lib/code.ts +++ b/packages/@aws-cdk/aws-lambda/lib/code.ts @@ -2,6 +2,7 @@ import * as s3 from '@aws-cdk/aws-s3'; import * as s3_assets from '@aws-cdk/aws-s3-assets'; import * as cdk from '@aws-cdk/core'; import * as fs from 'fs'; +import * as nodePath from 'path'; import { BundlingDockerImage, DockerVolume } from './bundling'; import { Function } from './function'; @@ -183,9 +184,9 @@ export interface BundlingOptions { readonly command: string[]; /** - * Docker volumes to mount. + * Additional Docker volumes to mount. * - * @default - The path to the asset file or directory is mounted at /asset + * @default - no additional volumes are mounted */ readonly volumes?: DockerVolume[]; @@ -199,9 +200,18 @@ export interface BundlingOptions { /** * Working directory inside the container. * - * @default - the `containerPath` of the first mounted volume. + * @default /src */ readonly workingDirectory?: string; + + /** + * Bundle directory. Subdirectories named after the asset path will be + * created in this directory and mounted at `/bundle` in the container. + * Should be added to your `.gitignore`. + * + * @default .bundle next to the asset directory + */ + readonly bundleDirectory?: string; } /** @@ -210,6 +220,9 @@ export interface BundlingOptions { export interface AssetCodeOptions extends s3_assets.AssetOptions { /** * Bundle the Lambda code by executing a command in a Docker container. + * The asset path will be mounted at `/src`. The Docker container is + * responsible for putting content at `/bundle`. The content at `/bundle` + * will be zipped and used as Lambda code. * * @default - asset path is zipped as is */ @@ -231,17 +244,30 @@ export class AssetCode extends Code { } public bind(scope: cdk.Construct): CodeConfig { + let bundleAssetPath: string | undefined; + if (this.options.bundling) { - // We are going to mount it, so ensure it exists - if (!fs.existsSync(this.path)) { - fs.mkdirSync(this.path); + // Create the directory for the bundle next to the asset path + const bundlePath = nodePath.join(nodePath.dirname(this.path), this.options.bundling.bundleDirectory ?? '.bundle'); + if (!fs.existsSync(bundlePath)) { + fs.mkdirSync(bundlePath); } - const volumes = this.options.bundling.volumes ?? [ + bundleAssetPath = nodePath.join(bundlePath, nodePath.basename(this.path)); + if (!fs.existsSync(bundleAssetPath)) { + fs.mkdirSync(bundleAssetPath); + } + + const volumes = [ { hostPath: this.path, - containerPath: '/asset', + containerPath: '/src', + }, + { + hostPath: bundleAssetPath, + containerPath: '/bundle', }, + ...this.options.bundling.volumes ?? [], ]; let image: BundlingDockerImage; @@ -259,14 +285,14 @@ export class AssetCode extends Code { command: this.options.bundling.command, volumes, environment: this.options.bundling.environment, - workingDirectory: this.options.bundling.workingDirectory ?? volumes[0].containerPath, + workingDirectory: this.options.bundling.workingDirectory ?? '/src', }); } // If the same AssetCode is used multiple times, retain only the first instantiation. if (!this.asset) { this.asset = new s3_assets.Asset(scope, 'Code', { - path: this.path, + path: bundleAssetPath ?? this.path, ...this.options, }); } diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.expected.json b/packages/@aws-cdk/aws-lambda/test/integ.bundling.expected.json index ecc3743fe645e..aa5a63c7a3c3d 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.bundling.expected.json +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.expected.json @@ -36,7 +36,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9S3BucketCB34B9C8" + "Ref": "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3Bucket6365D8AA" }, "S3Key": { "Fn::Join": [ @@ -49,7 +49,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9S3VersionKey037CDB93" + "Ref": "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3VersionKey14A1DBA7" } ] } @@ -62,7 +62,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9S3VersionKey037CDB93" + "Ref": "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3VersionKey14A1DBA7" } ] } @@ -87,17 +87,17 @@ } }, "Parameters": { - "AssetParameters32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9S3BucketCB34B9C8": { + "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3Bucket6365D8AA": { "Type": "String", - "Description": "S3 bucket for asset \"32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9\"" + "Description": "S3 bucket for asset \"0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cd\"" }, - "AssetParameters32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9S3VersionKey037CDB93": { + "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3VersionKey14A1DBA7": { "Type": "String", - "Description": "S3 key for asset version \"32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9\"" + "Description": "S3 key for asset version \"0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cd\"" }, - "AssetParameters32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9ArtifactHash24D3E285": { + "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdArtifactHashEEC2ED67": { "Type": "String", - "Description": "Artifact hash for asset \"32d2c35d0f19dda9c42556e7ee0cc5db58164e403d91dca376abeb614fb01bd9\"" + "Description": "Artifact hash for asset \"0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cd\"" } }, "Outputs": { diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts index 74df35f3d092c..fa0ae8fa95dfb 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts @@ -17,17 +17,15 @@ class TestStack extends Stack { const fn = new lambda.Function(this, 'Function', { code: lambda.Code.fromAsset(assetPath, { bundling: { - image: lambda.BundlingDockerImage.fromRegistry('python:3.6'), command: [ - 'pip', 'install', - '-r', 'requirements.txt', - '-t', '.', + 'bash', '-c', ` + pip install -r /src/requirements.txt -t /bundle && + rsync -r /src/ /bundle + `, ], }, // Python dependencies do not give a stable hash - sourceHash: FileSystem.fingerprint(assetPath, { - exclude: ['*', '!index.py', '!requirements.txt'], - }), + sourceHash: FileSystem.fingerprint(assetPath), }), runtime: lambda.Runtime.PYTHON_3_6, handler: 'index.handler', diff --git a/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/.gitignore b/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/.gitignore deleted file mode 100644 index 7e1de670e94bb..0000000000000 --- a/packages/@aws-cdk/aws-lambda/test/python-lambda-handler/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!.gitignore -!index.py -!requirements.txt diff --git a/packages/@aws-cdk/aws-lambda/test/test.code.ts b/packages/@aws-cdk/aws-lambda/test/test.code.ts index 8ec5a7ad6a0dc..7817f2cd87cbe 100644 --- a/packages/@aws-cdk/aws-lambda/test/test.code.ts +++ b/packages/@aws-cdk/aws-lambda/test/test.code.ts @@ -120,10 +120,11 @@ export = { // THEN test.ok(spawnSyncStub.calledWith('docker', [ 'run', '--rm', - '-v', `${dockerAssetPath}:/asset`, + '-v', `${dockerAssetPath}:/src`, + '-v', `.bundle/${dockerAssetPath}:/bundle`, '--env', 'VAR1=value1', '--env', 'VAR2=value2', - '-w', '/asset', + '-w', '/src', 'alpine', ...command, ]), 'docker run not called with expected args'); @@ -161,8 +162,9 @@ export = { // THEN test.ok(spawnSyncStub.calledWith('docker', [ 'run', '--rm', - '-v', `${dockerAssetPath}:/asset`, - '-w', '/asset', + '-v', `${dockerAssetPath}:/src`, + '-v', `.bundle/${dockerAssetPath}:/bundle`, + '-w', '/src', 'lambci/lambda:build-nodejs12.x', ...command, ]), 'docker run not called with expected args'); @@ -216,8 +218,9 @@ export = { test.ok(spawnSyncStub.calledWith('docker', [ 'run', '--rm', - '-v', `${dockerAssetPath}:/asset`, - '-w', '/asset', + '-v', `${dockerAssetPath}:/src`, + '-v', `.bundle/${dockerAssetPath}:/bundle`, + '-w', '/src', imageId, ...command, ]), 'docker run not called with expected args'); From 3a10273ff47632eaa7d12f3ae87df9b238be7f8e Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 18 May 2020 17:49:27 +0200 Subject: [PATCH 13/53] /src is the workdir --- packages/@aws-cdk/aws-lambda/test/integ.bundling.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts index fa0ae8fa95dfb..0146e886cc054 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts @@ -19,8 +19,8 @@ class TestStack extends Stack { bundling: { command: [ 'bash', '-c', ` - pip install -r /src/requirements.txt -t /bundle && - rsync -r /src/ /bundle + pip install -r requirements.txt -t /bundle && + rsync -r . /bundle `, ], }, From 97c81d4a418906c89e9bce2d4b0e050c0c4bfad4 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 18 May 2020 17:50:51 +0200 Subject: [PATCH 14/53] README --- packages/@aws-cdk/aws-lambda/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index 531a5d432ad86..cd90386099a54 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -268,8 +268,8 @@ new lambda.Function(this, 'Function', { bundling: { // Docker image defaults to the lambci/lambda build image for the function's runtime command: [ 'bash', '-c', ` - pip install -r /src/requirements.txt -t /bundle && - rsync -r /src/ /bundle + pip install -r requirements.txt -t /bundle && + rsync -r . /bundle `, ], }, From f5011d30c611fcf146c398394afe106d0ec3e013 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Wed, 20 May 2020 13:24:55 +0200 Subject: [PATCH 15/53] move to s3-assets --- packages/@aws-cdk/aws-lambda/README.md | 6 +- packages/@aws-cdk/aws-lambda/lib/code.ts | 120 +----- packages/@aws-cdk/aws-lambda/lib/index.ts | 1 - packages/@aws-cdk/aws-lambda/lib/runtime.ts | 11 + .../aws-lambda/test/integ.bundling.ts | 1 + .../@aws-cdk/aws-lambda/test/test.code.ts | 148 -------- packages/@aws-cdk/aws-s3-assets/.gitignore | 3 + packages/@aws-cdk/aws-s3-assets/.npmignore | 1 + .../@aws-cdk/aws-s3-assets/jest.config.js | 2 + packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 97 ++++- .../lib/bundling.ts | 7 +- packages/@aws-cdk/aws-s3-assets/lib/index.ts | 1 + packages/@aws-cdk/aws-s3-assets/package.json | 14 +- .../@aws-cdk/aws-s3-assets/test/asset.test.ts | 328 ++++++++++++++++ .../aws-s3-assets/test/bundling.test.ts | 141 +++++++ .../@aws-cdk/aws-s3-assets/test/test.asset.ts | 355 ------------------ 16 files changed, 599 insertions(+), 637 deletions(-) create mode 100644 packages/@aws-cdk/aws-s3-assets/jest.config.js rename packages/@aws-cdk/{aws-lambda => aws-s3-assets}/lib/bundling.ts (95%) create mode 100644 packages/@aws-cdk/aws-s3-assets/test/asset.test.ts create mode 100644 packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts delete mode 100644 packages/@aws-cdk/aws-s3-assets/test/test.asset.ts diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index cd90386099a54..d89cdc7b79155 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -265,7 +265,8 @@ Example with Python: ```ts new lambda.Function(this, 'Function', { code: lambda.Code.fromAsset(path.join(__dirname, 'my-python-handler'), { - bundling: { // Docker image defaults to the lambci/lambda build image for the function's runtime + bundling: { + image: lambda.Runtime.PYTHON_3_6.bundlingDockerImage, command: [ 'bash', '-c', ` pip install -r requirements.txt -t /bundle && @@ -278,8 +279,7 @@ new lambda.Function(this, 'Function', { handler: 'index.handler', }); ``` -The Docker image defaults to the [lambci/lambda](https://hub.docker.com/r/lambci/lambda/) build -image for the function's runtime. +Runtimes expose a `bundlingDockerImage` property that points to the [lambci/lambda](https://hub.docker.com/r/lambci/lambda/) build image. Use `lambda.DockerImage.fromRegistry(image)` to use an existing image or `lambda.DockerImage.fromAsset(path)` to build a specific image: diff --git a/packages/@aws-cdk/aws-lambda/lib/code.ts b/packages/@aws-cdk/aws-lambda/lib/code.ts index 3b50dc528d77f..f43b4dd076d5e 100644 --- a/packages/@aws-cdk/aws-lambda/lib/code.ts +++ b/packages/@aws-cdk/aws-lambda/lib/code.ts @@ -1,10 +1,6 @@ import * as s3 from '@aws-cdk/aws-s3'; import * as s3_assets from '@aws-cdk/aws-s3-assets'; import * as cdk from '@aws-cdk/core'; -import * as fs from 'fs'; -import * as nodePath from 'path'; -import { BundlingDockerImage, DockerVolume } from './bundling'; -import { Function } from './function'; export abstract class Code { /** @@ -44,7 +40,7 @@ export abstract class Code { * * @param path Either a directory with the Lambda code bundle or a .zip file */ - public static fromAsset(path: string, options?: AssetCodeOptions): AssetCode { + public static fromAsset(path: string, options?: s3_assets.AssetOptions): AssetCode { return new AssetCode(path, options); } @@ -164,71 +160,6 @@ export class InlineCode extends Code { } } -/** - * Bundling options - */ -export interface BundlingOptions { - /** - * The Docker image where the command will run. - * - * @default - The lambci/lambda build image for the function's runtime - * https://hub.docker.com/r/lambci/lambda/ - */ - readonly image?: BundlingDockerImage; - - /** - * The command to run in the container. - * - * @example ['npm', 'install'] - */ - readonly command: string[]; - - /** - * Additional Docker volumes to mount. - * - * @default - no additional volumes are mounted - */ - readonly volumes?: DockerVolume[]; - - /** - * The environment variables to pass to the container. - * - * @default - no environment variables. - */ - readonly environment?: { [key: string]: string; }; - - /** - * Working directory inside the container. - * - * @default /src - */ - readonly workingDirectory?: string; - - /** - * Bundle directory. Subdirectories named after the asset path will be - * created in this directory and mounted at `/bundle` in the container. - * Should be added to your `.gitignore`. - * - * @default .bundle next to the asset directory - */ - readonly bundleDirectory?: string; -} - -/** - * Asset code options - */ -export interface AssetCodeOptions extends s3_assets.AssetOptions { - /** - * Bundle the Lambda code by executing a command in a Docker container. - * The asset path will be mounted at `/src`. The Docker container is - * responsible for putting content at `/bundle`. The content at `/bundle` - * will be zipped and used as Lambda code. - * - * @default - asset path is zipped as is - */ - readonly bundling?: BundlingOptions; -} - /** * Lambda code from a local directory. */ @@ -239,60 +170,15 @@ export class AssetCode extends Code { /** * @param path The path to the asset file or directory. */ - constructor(public readonly path: string, private readonly options: AssetCodeOptions = {}) { + constructor(public readonly path: string, private readonly options: s3_assets.AssetOptions = {}) { super(); } public bind(scope: cdk.Construct): CodeConfig { - let bundleAssetPath: string | undefined; - - if (this.options.bundling) { - // Create the directory for the bundle next to the asset path - const bundlePath = nodePath.join(nodePath.dirname(this.path), this.options.bundling.bundleDirectory ?? '.bundle'); - if (!fs.existsSync(bundlePath)) { - fs.mkdirSync(bundlePath); - } - - bundleAssetPath = nodePath.join(bundlePath, nodePath.basename(this.path)); - if (!fs.existsSync(bundleAssetPath)) { - fs.mkdirSync(bundleAssetPath); - } - - const volumes = [ - { - hostPath: this.path, - containerPath: '/src', - }, - { - hostPath: bundleAssetPath, - containerPath: '/bundle', - }, - ...this.options.bundling.volumes ?? [], - ]; - - let image: BundlingDockerImage; - if (this.options.bundling.image) { - image = this.options.bundling.image; - } else { - if (scope instanceof Function) { - image = BundlingDockerImage.fromRegistry(`lambci/lambda:build-${scope.runtime.name}`); - } else { - throw new Error('Cannot derive default bundling image from scope. Please specify an image.'); - } - } - - image.run({ - command: this.options.bundling.command, - volumes, - environment: this.options.bundling.environment, - workingDirectory: this.options.bundling.workingDirectory ?? '/src', - }); - } - // If the same AssetCode is used multiple times, retain only the first instantiation. if (!this.asset) { this.asset = new s3_assets.Asset(scope, 'Code', { - path: bundleAssetPath ?? this.path, + path: this.path, ...this.options, }); } diff --git a/packages/@aws-cdk/aws-lambda/lib/index.ts b/packages/@aws-cdk/aws-lambda/lib/index.ts index e77a1880d808a..b494e924c604a 100644 --- a/packages/@aws-cdk/aws-lambda/lib/index.ts +++ b/packages/@aws-cdk/aws-lambda/lib/index.ts @@ -12,7 +12,6 @@ export * from './event-source'; export * from './event-source-mapping'; export * from './destination'; export * from './event-invoke-config'; -export * from './bundling'; export * from './log-retention'; diff --git a/packages/@aws-cdk/aws-lambda/lib/runtime.ts b/packages/@aws-cdk/aws-lambda/lib/runtime.ts index ffa111ca4509b..b2d1ec4083a7d 100644 --- a/packages/@aws-cdk/aws-lambda/lib/runtime.ts +++ b/packages/@aws-cdk/aws-lambda/lib/runtime.ts @@ -1,3 +1,5 @@ +import * as assets from '@aws-cdk/aws-s3-assets'; + export interface LambdaRuntimeProps { /** * Whether the ``ZipFile`` (aka inline code) property can be used with this runtime. @@ -154,10 +156,19 @@ export class Runtime { */ public readonly family?: RuntimeFamily; + /** + * The bundling Docker image for this runtime. + * Points to the lambdci/lambda build image for this runtime. + * + * @see https://hub.docker.com/r/lambci/lambda/ + */ + public readonly bundlingDockerImage: assets.BundlingDockerImage; + constructor(name: string, family?: RuntimeFamily, props: LambdaRuntimeProps = { }) { this.name = name; this.supportsInlineCode = !!props.supportsInlineCode; this.family = family; + this.bundlingDockerImage = assets.BundlingDockerImage.fromRegistry(`lambci/lambda:build-${name}`); Runtime.ALL.push(this); } diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts index 0146e886cc054..0c791b6615071 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts @@ -17,6 +17,7 @@ class TestStack extends Stack { const fn = new lambda.Function(this, 'Function', { code: lambda.Code.fromAsset(assetPath, { bundling: { + image: lambda.Runtime.PYTHON_3_6.bundlingDockerImage, command: [ 'bash', '-c', ` pip install -r requirements.txt -t /bundle && diff --git a/packages/@aws-cdk/aws-lambda/test/test.code.ts b/packages/@aws-cdk/aws-lambda/test/test.code.ts index 7817f2cd87cbe..98ac8e0364cac 100644 --- a/packages/@aws-cdk/aws-lambda/test/test.code.ts +++ b/packages/@aws-cdk/aws-lambda/test/test.code.ts @@ -1,10 +1,8 @@ import { expect, haveResource, haveResourceLike, ResourcePart } from '@aws-cdk/assert'; import * as cdk from '@aws-cdk/core'; import * as cxapi from '@aws-cdk/cx-api'; -import * as child_process from 'child_process'; import { Test } from 'nodeunit'; import * as path from 'path'; -import * as sinon from 'sinon'; import * as lambda from '../lib'; // tslint:disable:no-string-literal @@ -84,152 +82,6 @@ export = { }, ResourcePart.CompleteDefinition)); test.done(); }, - - 'docker image'(test: Test) { - // GIVEN - const stack = new cdk.Stack(); - - const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - }); - - const dockerAssetPath = 'asset-path'; - const command = ['this', 'is', 'a', 'build', 'command']; - - // WHEN - new lambda.Function(stack, 'Fn', { - handler: 'foom', - runtime: lambda.Runtime.NODEJS_12_X, - code: lambda.Code.fromAsset(dockerAssetPath, { - bundling: { - image: lambda.BundlingDockerImage.fromRegistry('alpine'), - environment: { - VAR1: 'value1', - VAR2: 'value2', - }, - command, - }, - }), - }); - - // THEN - test.ok(spawnSyncStub.calledWith('docker', [ - 'run', '--rm', - '-v', `${dockerAssetPath}:/src`, - '-v', `.bundle/${dockerAssetPath}:/bundle`, - '--env', 'VAR1=value1', - '--env', 'VAR2=value2', - '-w', '/src', - 'alpine', - ...command, - ]), 'docker run not called with expected args'); - - spawnSyncStub.restore(); - - test.done(); - }, - - 'docker image default to lambci'(test: Test) { - // GIVEN - const stack = new cdk.Stack(); - - const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - }); - - const dockerAssetPath = 'asset-path'; - const command = ['this', 'is', 'a', 'build', 'command']; - - // WHEN - new lambda.Function(stack, 'Fn', { - handler: 'foom', - runtime: lambda.Runtime.NODEJS_12_X, - code: lambda.Code.fromAsset(dockerAssetPath, { - bundling: { command }, - }), - }); - - // THEN - test.ok(spawnSyncStub.calledWith('docker', [ - 'run', '--rm', - '-v', `${dockerAssetPath}:/src`, - '-v', `.bundle/${dockerAssetPath}:/bundle`, - '-w', '/src', - 'lambci/lambda:build-nodejs12.x', - ...command, - ]), 'docker run not called with expected args'); - - spawnSyncStub.restore(); - - test.done(); - }, - - 'docker build'(test: Test) { - // GIVEN - const stack = new cdk.Stack(); - - const imageId = 'abcdef123456'; - const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from(`Successfully built ${imageId}`), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - }); - - const dockerPath = 'docker-path'; - const testArg = 'cdk-test'; - const dockerAssetPath = 'asset-path'; - const command = ['this', 'is', 'a', 'build', 'command']; - - // WHEN - new lambda.Function(stack, 'Fn', { - handler: 'foom', - runtime: lambda.Runtime.NODEJS_12_X, - code: lambda.Code.fromAsset(dockerAssetPath, { - bundling: { - image: lambda.BundlingDockerImage.fromAsset(dockerPath, { - buildArgs: { - TEST_ARG: testArg, - }, - }), - command, - }, - }), - }); - - // THEN - test.ok(spawnSyncStub.calledWith('docker', [ - 'build', - '--build-arg', `TEST_ARG=${testArg}`, - dockerPath, - ]), 'docker build not called with expected args'); - - test.ok(spawnSyncStub.calledWith('docker', [ - 'run', '--rm', - '-v', `${dockerAssetPath}:/src`, - '-v', `.bundle/${dockerAssetPath}:/bundle`, - '-w', '/src', - imageId, - ...command, - ]), 'docker run not called with expected args'); - - spawnSyncStub.restore(); - - test.done(); - - }, }, 'lambda.Code.fromCfnParameters': { diff --git a/packages/@aws-cdk/aws-s3-assets/.gitignore b/packages/@aws-cdk/aws-s3-assets/.gitignore index 84107ada8a317..9612a7e9c2696 100644 --- a/packages/@aws-cdk/aws-s3-assets/.gitignore +++ b/packages/@aws-cdk/aws-s3-assets/.gitignore @@ -15,3 +15,6 @@ nyc.config.js .LAST_PACKAGE *.snk !.eslintrc.js +!jest.config.js + +.bundle diff --git a/packages/@aws-cdk/aws-s3-assets/.npmignore b/packages/@aws-cdk/aws-s3-assets/.npmignore index 174864d493a79..4e4f173de8358 100644 --- a/packages/@aws-cdk/aws-s3-assets/.npmignore +++ b/packages/@aws-cdk/aws-s3-assets/.npmignore @@ -19,3 +19,4 @@ dist tsconfig.json .eslintrc.js +jest.config.js diff --git a/packages/@aws-cdk/aws-s3-assets/jest.config.js b/packages/@aws-cdk/aws-s3-assets/jest.config.js new file mode 100644 index 0000000000000..cd664e1d069e5 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-assets/jest.config.js @@ -0,0 +1,2 @@ +const baseConfig = require('../../../tools/cdk-build-tools/config/jest.config'); +module.exports = baseConfig; diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 5c4b6e6cb3eb9..64866e62165bc 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -5,11 +5,62 @@ import * as cdk from '@aws-cdk/core'; import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs'; import * as path from 'path'; +import { BundlingDockerImage, DockerVolume } from './bundling'; const ARCHIVE_EXTENSIONS = [ '.zip', '.jar' ]; +const BUNDLING_INPUT_DIR = '/input'; +const BUNDLING_OUTPUT_DIR = '/output'; -export interface AssetOptions extends assets.CopyOptions { +/** + * Bundling options + */ +export interface BundlingOptions { + /** + * The Docker image where the command will run. + */ + readonly image: BundlingDockerImage; + + /** + * The command to run in the container. + * + * @example ['npm', 'install'] + * + * @default - run the command defined in the image + */ + readonly command?: string[]; + + /** + * Additional Docker volumes to mount. + * + * @default - no additional volumes are mounted + */ + readonly volumes?: DockerVolume[]; + + /** + * The environment variables to pass to the container. + * + * @default - no environment variables. + */ + readonly environment?: { [key: string]: string; }; + + /** + * Working directory inside the container. + * + * @default /input + */ + readonly workingDirectory?: string; + /** + * Bundle directory. Subdirectories named after the asset path will be + * created in this directory and mounted at `/output` in the container. + * Should be added to your `.gitignore`. + * + * @default .bundle next to the asset directory + */ + readonly bundleDirectory?: string; +} + +export interface AssetOptions extends assets.CopyOptions { /** * A list of principals that should be able to read this asset from S3. * You can use `asset.grantRead(principal)` to grant read permissions later. @@ -33,6 +84,16 @@ export interface AssetOptions extends assets.CopyOptions { * @experimental */ readonly sourceHash?: string; + + /** + * Bundle the asset by executing a command in a Docker container. + * The asset path will be mounted at `/input`. The Docker container is + * responsible for putting content at `/output`. The content at `/output` + * will be zipped and used as the asset. + * + * @default - asset path is zipped as is + */ + readonly bundling?: BundlingOptions; } export interface AssetProps extends AssetOptions { @@ -103,6 +164,40 @@ export class Asset extends cdk.Construct implements assets.IAsset { constructor(scope: cdk.Construct, id: string, props: AssetProps) { super(scope, id); + // Bundling + let bundleAssetPath: string | undefined; + if (props.bundling) { + // Create the directory for the bundle next to the asset path + const bundlePath = path.join(path.dirname(props.path), props.bundling.bundleDirectory ?? '.bundle'); + if (!fs.existsSync(bundlePath)) { + fs.mkdirSync(bundlePath); + } + + bundleAssetPath = path.join(bundlePath, path.basename(props.path)); + if (!fs.existsSync(bundleAssetPath)) { + fs.mkdirSync(bundleAssetPath); + } + + const volumes = [ + { + hostPath: props.path, + containerPath: BUNDLING_INPUT_DIR, + }, + { + hostPath: bundleAssetPath, + containerPath: BUNDLING_OUTPUT_DIR, + }, + ...props.bundling.volumes ?? [], + ]; + + props.bundling.image.run({ + command: props.bundling.command, + volumes, + environment: props.bundling.environment, + workingDirectory: props.bundling.workingDirectory ?? BUNDLING_INPUT_DIR, + }); + } + // stage the asset source (conditionally). const staging = new assets.Staging(this, 'Stage', { ...props, diff --git a/packages/@aws-cdk/aws-lambda/lib/bundling.ts b/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts similarity index 95% rename from packages/@aws-cdk/aws-lambda/lib/bundling.ts rename to packages/@aws-cdk/aws-s3-assets/lib/bundling.ts index 57e90ad889634..9313f7a05d0ca 100644 --- a/packages/@aws-cdk/aws-lambda/lib/bundling.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts @@ -21,8 +21,10 @@ export interface DockerVolume { export interface DockerRunOptions { /** * The command to run in the container. + * + * @default - run the command defined in the image */ - readonly command: string[]; + readonly command?: string[]; /** * Docker volumes to mount. @@ -106,6 +108,7 @@ export class BundlingDockerImage { public run(options: DockerRunOptions) { const volumes = options.volumes || []; const environment = options.environment || {}; + const command = options.command || []; const dockerArgs: string[] = [ 'run', '--rm', @@ -115,7 +118,7 @@ export class BundlingDockerImage { ? ['-w', options.workingDirectory] : [], this.image, - ...options.command, + ...command, ]; exec('docker', dockerArgs); diff --git a/packages/@aws-cdk/aws-s3-assets/lib/index.ts b/packages/@aws-cdk/aws-s3-assets/lib/index.ts index ea2719dd83bd3..3ea08e61cb5b9 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/index.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/index.ts @@ -1 +1,2 @@ export * from './asset'; +export * from './bundling'; diff --git a/packages/@aws-cdk/aws-s3-assets/package.json b/packages/@aws-cdk/aws-s3-assets/package.json index ff1eb0933ce36..3b8fe5bdebded 100644 --- a/packages/@aws-cdk/aws-s3-assets/package.json +++ b/packages/@aws-cdk/aws-s3-assets/package.json @@ -45,6 +45,9 @@ "build+test": "npm run build && npm test", "compat": "cdk-compat" }, + "cdk-build": { + "jest": true + }, "keywords": [ "aws", "cdk", @@ -60,16 +63,10 @@ "license": "Apache-2.0", "devDependencies": { "@aws-cdk/assert": "0.0.0", - "@types/nodeunit": "^0.0.31", - "@types/sinon": "^9.0.3", - "aws-cdk": "0.0.0", "cdk-build-tools": "0.0.0", "cdk-integ-tools": "0.0.0", - "nodeunit": "^0.11.3", "pkglint": "0.0.0", - "sinon": "^9.0.2", - "@aws-cdk/cloud-assembly-schema": "0.0.0", - "ts-mock-imports": "^1.3.0" + "@aws-cdk/cloud-assembly-schema": "0.0.0" }, "dependencies": { "@aws-cdk/assets": "0.0.0", @@ -93,9 +90,6 @@ }, "stability": "experimental", "maturity": "experimental", - "nyc": { - "statements": 75 - }, "awslint": { "exclude": [ "docs-public-apis:@aws-cdk/aws-s3-assets.AssetOptions", diff --git a/packages/@aws-cdk/aws-s3-assets/test/asset.test.ts b/packages/@aws-cdk/aws-s3-assets/test/asset.test.ts new file mode 100644 index 0000000000000..4da45143c59f8 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-assets/test/asset.test.ts @@ -0,0 +1,328 @@ +import { ResourcePart, SynthUtils } from '@aws-cdk/assert'; +import '@aws-cdk/assert/jest'; +import * as iam from '@aws-cdk/aws-iam'; +import * as cxschema from '@aws-cdk/cloud-assembly-schema'; +import * as cdk from '@aws-cdk/core'; +import * as cxapi from '@aws-cdk/cx-api'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { Asset } from '../lib/asset'; + +const SAMPLE_ASSET_DIR = path.join(__dirname, 'sample-asset-directory'); + +test('simple use case', () => { + const app = new cdk.App({ + context: { + [cxapi.DISABLE_ASSET_STAGING_CONTEXT]: 'true', + }, + }); + const stack = new cdk.Stack(app, 'MyStack'); + new Asset(stack, 'MyAsset', { + path: SAMPLE_ASSET_DIR, + }); + + // verify that metadata contains an "aws:cdk:asset" entry with + // the correct information + const entry = stack.node.metadata.find(m => m.type === 'aws:cdk:asset'); + expect(entry).toBeTruthy(); + + // verify that now the template contains parameters for this asset + const session = app.synth(); + + expect(stack.resolve(entry!.data)).toEqual({ + path: SAMPLE_ASSET_DIR, + id: '6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', + packaging: 'zip', + sourceHash: '6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', + s3BucketParameter: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3Bucket50B5A10B', + s3KeyParameter: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3VersionKey1F7D75F9', + artifactHashParameter: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2ArtifactHash220DE9BD', + }); + + const template = JSON.parse(fs.readFileSync(path.join(session.directory, 'MyStack.template.json'), 'utf-8')); + + expect(template.Parameters.AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3Bucket50B5A10B.Type).toBe('String'); + expect(template.Parameters.AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3VersionKey1F7D75F9.Type).toBe('String'); +}); + +test('verify that the app resolves tokens in metadata', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'my-stack'); + const dirPath = path.resolve(__dirname, 'sample-asset-directory'); + + new Asset(stack, 'MyAsset', { + path: dirPath, + }); + + const synth = app.synth().getStackByName(stack.stackName); + const meta = synth.manifest.metadata || {}; + expect(meta['/my-stack']).toBeTruthy(); + expect(meta['/my-stack'][0]).toBeTruthy(); + expect(meta['/my-stack'][0].data).toEqual({ + path: 'asset.6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', + id: '6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', + packaging: 'zip', + sourceHash: '6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', + s3BucketParameter: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3Bucket50B5A10B', + s3KeyParameter: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3VersionKey1F7D75F9', + artifactHashParameter: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2ArtifactHash220DE9BD', + }); +}); + +test('"file" assets', () => { + const stack = new cdk.Stack(); + const filePath = path.join(__dirname, 'file-asset.txt'); + new Asset(stack, 'MyAsset', { path: filePath }); + const entry = stack.node.metadata.find(m => m.type === 'aws:cdk:asset'); + expect(entry).toBeTruthy(); + + // synthesize first so "prepare" is called + const template = SynthUtils.synthesize(stack).template; + + expect(stack.resolve(entry!.data)).toEqual({ + path: 'asset.78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197.txt', + packaging: 'file', + id: '78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197', + sourceHash: '78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197', + s3BucketParameter: 'AssetParameters78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197S3Bucket2C60F94A', + s3KeyParameter: 'AssetParameters78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197S3VersionKey9482DC35', + artifactHashParameter: 'AssetParameters78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197ArtifactHash22BFFA67', + }); + + // verify that now the template contains parameters for this asset + expect(template.Parameters.AssetParameters78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197S3Bucket2C60F94A.Type).toBe('String'); + expect(template.Parameters.AssetParameters78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197S3VersionKey9482DC35.Type).toBe('String'); +}); + +test('"readers" or "grantRead" can be used to grant read permissions on the asset to a principal', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'MyUser'); + const group = new iam.Group(stack, 'MyGroup'); + + const asset = new Asset(stack, 'MyAsset', { + path: path.join(__dirname, 'sample-asset-directory'), + readers: [ user ], + }); + + asset.grantRead(group); + + expect(stack).toHaveResource('AWS::IAM::Policy', { + PolicyDocument: { + Version: '2012-10-17', + Statement: [ + { + Action: ['s3:GetObject*', 's3:GetBucket*', 's3:List*'], + Effect: 'Allow', + Resource: [ + { 'Fn::Join': ['', ['arn:', {Ref: 'AWS::Partition'}, ':s3:::', {Ref: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3Bucket50B5A10B'} ] ] }, + { 'Fn::Join': ['', [ 'arn:', {Ref: 'AWS::Partition'}, ':s3:::', {Ref: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3Bucket50B5A10B'}, '/*' ] ] }, + ], + }, + ], + }, + }); +}); + +test('fails if directory not found', () => { + const stack = new cdk.Stack(); + expect(() => new Asset(stack, 'MyDirectory', { + path: '/path/not/found/' + Math.random() * 999999, + })).toThrow(); +}); + +test('multiple assets under the same parent', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + expect(() => new Asset(stack, 'MyDirectory1', { path: path.join(__dirname, 'sample-asset-directory') })).not.toThrow(); + expect(() => new Asset(stack, 'MyDirectory2', { path: path.join(__dirname, 'sample-asset-directory') })).not.toThrow(); +}); + +test('isZipArchive indicates if the asset represents a .zip file (either explicitly or via ZipDirectory packaging)', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + const nonZipAsset = new Asset(stack, 'NonZipAsset', { + path: path.join(__dirname, 'sample-asset-directory', 'sample-asset-file.txt'), + }); + + const zipDirectoryAsset = new Asset(stack, 'ZipDirectoryAsset', { + path: path.join(__dirname, 'sample-asset-directory'), + }); + + const zipFileAsset = new Asset(stack, 'ZipFileAsset', { + path: path.join(__dirname, 'sample-asset-directory', 'sample-zip-asset.zip'), + }); + + const jarFileAsset = new Asset(stack, 'JarFileAsset', { + path: path.join(__dirname, 'sample-asset-directory', 'sample-jar-asset.jar'), + }); + + // THEN + expect(nonZipAsset.isZipArchive).toBe(false); + expect(zipDirectoryAsset.isZipArchive).toBe(true); + expect(zipFileAsset.isZipArchive).toBe(true); + expect(jarFileAsset.isZipArchive).toBe(true); +}); + +test('addResourceMetadata can be used to add CFN metadata to resources', () => { + // GIVEN + const stack = new cdk.Stack(); + stack.node.setContext(cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT, true); + + const location = path.join(__dirname, 'sample-asset-directory'); + const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); + const asset = new Asset(stack, 'MyAsset', { path: location }); + + // WHEN + asset.addResourceMetadata(resource, 'PropName'); + + // THEN + expect(stack).toHaveResource('My::Resource::Type', { + Metadata: { + 'aws:asset:path': 'asset.6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', + 'aws:asset:property': 'PropName', + }, + }, ResourcePart.CompleteDefinition); +}); + +test('asset metadata is only emitted if ASSET_RESOURCE_METADATA_ENABLED_CONTEXT is defined', () => { + // GIVEN + const stack = new cdk.Stack(); + + const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); + const asset = new Asset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); + + // WHEN + asset.addResourceMetadata(resource, 'PropName'); + + // THEN + expect(stack).not.toHaveResource('My::Resource::Type', { + Metadata: { + 'aws:asset:path': SAMPLE_ASSET_DIR, + 'aws:asset:property': 'PropName', + }, + }, ResourcePart.CompleteDefinition); +}); + +describe('staging', () => { + test('copy file assets under /${fingerprint}.ext', () => { + const tempdir = mkdtempSync(); + process.chdir(tempdir); // change current directory to somewhere in /tmp + + // GIVEN + const app = new cdk.App({ outdir: tempdir }); + const stack = new cdk.Stack(app, 'stack'); + + // WHEN + new Asset(stack, 'ZipFile', { + path: path.join(SAMPLE_ASSET_DIR, 'sample-zip-asset.zip'), + }); + + new Asset(stack, 'TextFile', { + path: path.join(SAMPLE_ASSET_DIR, 'sample-asset-file.txt'), + }); + + // THEN + app.synth(); + expect(fs.existsSync(tempdir)).toBe(true); + expect(fs.existsSync(path.join(tempdir, 'asset.a7a79cdf84b802ea8b198059ff899cffc095a1b9606e919f98e05bf80779756b.zip'))).toBe(true); + }); + + test('copy directory under .assets/fingerprint/**', () => { + const tempdir = mkdtempSync(); + process.chdir(tempdir); // change current directory to somewhere in /tmp + + // GIVEN + const app = new cdk.App({ outdir: tempdir }); + const stack = new cdk.Stack(app, 'stack'); + + // WHEN + new Asset(stack, 'ZipDirectory', { + path: SAMPLE_ASSET_DIR, + }); + + // THEN + app.synth(); + expect(fs.existsSync(tempdir)).toBe(true); + const hash = 'asset.6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2'; + expect(fs.existsSync(path.join(tempdir, hash, 'sample-asset-file.txt'))).toBe(true); + expect(fs.existsSync(path.join(tempdir, hash, 'sample-jar-asset.jar'))).toBe(true); + expect(() => fs.readdirSync(tempdir)).not.toThrow(); + }); + + test('staging path is relative if the dir is below the working directory', () => { + // GIVEN + const tempdir = mkdtempSync(); + process.chdir(tempdir); // change current directory to somewhere in /tmp + + const staging = '.my-awesome-staging-directory'; + const app = new cdk.App({ + outdir: staging, + context: { + [cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT]: 'true', + }, + }); + + const stack = new cdk.Stack(app, 'stack'); + + const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); + const asset = new Asset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); + + // WHEN + asset.addResourceMetadata(resource, 'PropName'); + + const template = SynthUtils.synthesize(stack).template; + expect(template.Resources.MyResource.Metadata).toEqual({ + 'aws:asset:path': 'asset.6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', + 'aws:asset:property': 'PropName', + }); + }); + + test('if staging is disabled, asset path is absolute', () => { + // GIVEN + const staging = path.resolve(mkdtempSync()); + const app = new cdk.App({ + outdir: staging, + context: { + [cxapi.DISABLE_ASSET_STAGING_CONTEXT]: 'true', + [cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT]: 'true', + }, + }); + + const stack = new cdk.Stack(app, 'stack'); + + const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); + const asset = new Asset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); + + // WHEN + asset.addResourceMetadata(resource, 'PropName'); + + const template = SynthUtils.synthesize(stack).template; + expect(template.Resources.MyResource.Metadata).toEqual({ + 'aws:asset:path': SAMPLE_ASSET_DIR, + 'aws:asset:property': 'PropName', + }); + }); + + test('cdk metadata points to staged asset', () => { + // GIVEN + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'stack'); + new Asset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); + + // WHEN + const session = app.synth(); + const artifact = session.getStackByName(stack.stackName); + const metadata = artifact.manifest.metadata || {}; + const md = Object.values(metadata)[0]![0]!.data as cxschema.AssetMetadataEntry; + expect(md.path).toBe('asset.6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2'); + }); +}); + +function mkdtempSync() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'assets.test')); +} diff --git a/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts b/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts new file mode 100644 index 0000000000000..48b2e9a1df110 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts @@ -0,0 +1,141 @@ +import { Stack } from '@aws-cdk/core'; +import { spawnSync } from 'child_process'; +import * as path from 'path'; +import * as assets from '../lib'; + +const SAMPLE_ASSET_DIR = path.join(__dirname, 'sample-asset-directory'); + +jest.mock('child_process'); + +let stack: Stack; +beforeEach(() => { + stack = new Stack(); +}); + +test('bundling with image from registry', () => { + (spawnSync as jest.Mock).mockImplementation((): ReturnType => ({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + })); + + const command = ['this', 'is', 'a', 'build', 'command']; + const image = 'alpine'; + new assets.Asset(stack, 'Asset', { + path: SAMPLE_ASSET_DIR, + bundling: { + image: assets.BundlingDockerImage.fromRegistry(image), + environment: { + VAR1: 'value1', + VAR2: 'value2', + }, + command, + }, + }); + + expect(spawnSync).toHaveBeenCalledWith('docker', [ + 'run', '--rm', + '-v', `${SAMPLE_ASSET_DIR}:/input`, + '-v', expect.stringMatching(new RegExp(`${path.join('.bundle', path.basename(SAMPLE_ASSET_DIR))}:/output$`)), + '--env', 'VAR1=value1', + '--env', 'VAR2=value2', + '-w', '/input', + image, + ...command, + ]); +}); + +test('bundling with image from asset', () => { + const imageId = 'abcdef123456'; + (spawnSync as jest.Mock).mockImplementation((): ReturnType => ({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from(`Successfully built ${imageId}`), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + })); + + const dockerPath = 'docker-path'; + const testArg = 'cdk-test'; + new assets.Asset(stack, 'Asset', { + path: SAMPLE_ASSET_DIR, + bundling: { + image: assets.BundlingDockerImage.fromAsset(dockerPath, { + buildArgs: { + TEST_ARG: testArg, + }, + }), + }, + }); + + expect(spawnSync).toHaveBeenCalledWith('docker', [ + 'build', + '--build-arg', `TEST_ARG=${testArg}`, + dockerPath, + ]); + + expect(spawnSync).toHaveBeenCalledWith('docker', expect.arrayContaining([ + imageId, + ])); +}); + +test('throws if image id cannot be extracted from build output', () => { + (spawnSync as jest.Mock).mockImplementation((): ReturnType => ({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + })); + + const dockerPath = 'docker-path'; + expect(() => new assets.Asset(stack, 'Asset', { + path: SAMPLE_ASSET_DIR, + bundling: { + image: assets.BundlingDockerImage.fromAsset(dockerPath), + }, + })).toThrow(/Failed to extract image ID from Docker build output/); +}); + +test('throws in case of spawnSync error', () => { + const spawnSyncError = new Error('UnknownError'); + (spawnSync as jest.Mock).mockImplementation((): ReturnType => ({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + error: spawnSyncError, + })); + + expect(() => new assets.Asset(stack, 'Asset', { + path: SAMPLE_ASSET_DIR, + bundling: { + image: assets.BundlingDockerImage.fromRegistry('alpine'), + }, + })).toThrow(spawnSyncError); +}); + +test('throws if status is not 0', () => { + (spawnSync as jest.Mock).mockImplementation((): ReturnType => ({ + status: -1, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + })); + + expect(() => new assets.Asset(stack, 'Asset', { + path: SAMPLE_ASSET_DIR, + bundling: { + image: assets.BundlingDockerImage.fromRegistry('alpine'), + }, + })).toThrow(/^\[Status -1\]/); +}); diff --git a/packages/@aws-cdk/aws-s3-assets/test/test.asset.ts b/packages/@aws-cdk/aws-s3-assets/test/test.asset.ts deleted file mode 100644 index 68ef08d863d76..0000000000000 --- a/packages/@aws-cdk/aws-s3-assets/test/test.asset.ts +++ /dev/null @@ -1,355 +0,0 @@ -import { expect, haveResource, ResourcePart, SynthUtils } from '@aws-cdk/assert'; -import * as iam from '@aws-cdk/aws-iam'; -import * as cxschema from '@aws-cdk/cloud-assembly-schema'; -import * as cdk from '@aws-cdk/core'; -import * as cxapi from '@aws-cdk/cx-api'; -import * as fs from 'fs'; -import { Test } from 'nodeunit'; -import * as os from 'os'; -import * as path from 'path'; -import { Asset } from '../lib/asset'; - -// tslint:disable:max-line-length - -const SAMPLE_ASSET_DIR = path.join(__dirname, 'sample-asset-directory'); - -export = { - 'simple use case'(test: Test) { - const app = new cdk.App({ - context: { - [cxapi.DISABLE_ASSET_STAGING_CONTEXT]: 'true', - }, - }); - const stack = new cdk.Stack(app, 'MyStack'); - new Asset(stack, 'MyAsset', { - path: SAMPLE_ASSET_DIR, - }); - - // verify that metadata contains an "aws:cdk:asset" entry with - // the correct information - const entry = stack.node.metadata.find(m => m.type === 'aws:cdk:asset'); - test.ok(entry, 'found metadata entry'); - - // verify that now the template contains parameters for this asset - const session = app.synth(); - - test.deepEqual(stack.resolve(entry!.data), { - path: SAMPLE_ASSET_DIR, - id: '6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', - packaging: 'zip', - sourceHash: '6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', - s3BucketParameter: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3Bucket50B5A10B', - s3KeyParameter: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3VersionKey1F7D75F9', - artifactHashParameter: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2ArtifactHash220DE9BD', - }); - - const template = JSON.parse(fs.readFileSync(path.join(session.directory, 'MyStack.template.json'), 'utf-8')); - - test.equal(template.Parameters.AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3Bucket50B5A10B.Type, 'String'); - test.equal(template.Parameters.AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3VersionKey1F7D75F9.Type, 'String'); - - test.done(); - }, - - 'verify that the app resolves tokens in metadata'(test: Test) { - const app = new cdk.App(); - const stack = new cdk.Stack(app, 'my-stack'); - const dirPath = path.resolve(__dirname, 'sample-asset-directory'); - - new Asset(stack, 'MyAsset', { - path: dirPath, - }); - - const synth = app.synth().getStackByName(stack.stackName); - const meta = synth.manifest.metadata || {}; - test.ok(meta['/my-stack']); - test.ok(meta['/my-stack'][0]); - test.deepEqual(meta['/my-stack'][0].data, { - path: 'asset.6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', - id: '6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', - packaging: 'zip', - sourceHash: '6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', - s3BucketParameter: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3Bucket50B5A10B', - s3KeyParameter: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3VersionKey1F7D75F9', - artifactHashParameter: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2ArtifactHash220DE9BD', - }); - - test.done(); - }, - - '"file" assets'(test: Test) { - const stack = new cdk.Stack(); - const filePath = path.join(__dirname, 'file-asset.txt'); - new Asset(stack, 'MyAsset', { path: filePath }); - const entry = stack.node.metadata.find(m => m.type === 'aws:cdk:asset'); - test.ok(entry, 'found metadata entry'); - - // synthesize first so "prepare" is called - const template = SynthUtils.synthesize(stack).template; - - test.deepEqual(stack.resolve(entry!.data), { - path: 'asset.78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197.txt', - packaging: 'file', - id: '78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197', - sourceHash: '78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197', - s3BucketParameter: 'AssetParameters78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197S3Bucket2C60F94A', - s3KeyParameter: 'AssetParameters78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197S3VersionKey9482DC35', - artifactHashParameter: 'AssetParameters78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197ArtifactHash22BFFA67', - }); - - // verify that now the template contains parameters for this asset - test.equal(template.Parameters.AssetParameters78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197S3Bucket2C60F94A.Type, 'String'); - test.equal(template.Parameters.AssetParameters78add9eaf468dfa2191da44a7da92a21baba4c686cf6053d772556768ef21197S3VersionKey9482DC35.Type, 'String'); - - test.done(); - }, - - '"readers" or "grantRead" can be used to grant read permissions on the asset to a principal'(test: Test) { - const stack = new cdk.Stack(); - const user = new iam.User(stack, 'MyUser'); - const group = new iam.Group(stack, 'MyGroup'); - - const asset = new Asset(stack, 'MyAsset', { - path: path.join(__dirname, 'sample-asset-directory'), - readers: [ user ], - }); - - asset.grantRead(group); - - expect(stack).to(haveResource('AWS::IAM::Policy', { - PolicyDocument: { - Version: '2012-10-17', - Statement: [ - { - Action: ['s3:GetObject*', 's3:GetBucket*', 's3:List*'], - Effect: 'Allow', - Resource: [ - { 'Fn::Join': ['', ['arn:', {Ref: 'AWS::Partition'}, ':s3:::', {Ref: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3Bucket50B5A10B'} ] ] }, - { 'Fn::Join': ['', [ 'arn:', {Ref: 'AWS::Partition'}, ':s3:::', {Ref: 'AssetParameters6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2S3Bucket50B5A10B'}, '/*' ] ] }, - ], - }, - ], - }, - })); - - test.done(); - }, - 'fails if directory not found'(test: Test) { - const stack = new cdk.Stack(); - test.throws(() => new Asset(stack, 'MyDirectory', { - path: '/path/not/found/' + Math.random() * 999999, - })); - test.done(); - }, - - 'multiple assets under the same parent'(test: Test) { - // GIVEN - const stack = new cdk.Stack(); - - // WHEN - new Asset(stack, 'MyDirectory1', { path: path.join(__dirname, 'sample-asset-directory') }); - new Asset(stack, 'MyDirectory2', { path: path.join(__dirname, 'sample-asset-directory') }); - - // THEN: no error - - test.done(); - }, - - 'isZipArchive indicates if the asset represents a .zip file (either explicitly or via ZipDirectory packaging)'(test: Test) { - // GIVEN - const stack = new cdk.Stack(); - - // WHEN - const nonZipAsset = new Asset(stack, 'NonZipAsset', { - path: path.join(__dirname, 'sample-asset-directory', 'sample-asset-file.txt'), - }); - - const zipDirectoryAsset = new Asset(stack, 'ZipDirectoryAsset', { - path: path.join(__dirname, 'sample-asset-directory'), - }); - - const zipFileAsset = new Asset(stack, 'ZipFileAsset', { - path: path.join(__dirname, 'sample-asset-directory', 'sample-zip-asset.zip'), - }); - - const jarFileAsset = new Asset(stack, 'JarFileAsset', { - path: path.join(__dirname, 'sample-asset-directory', 'sample-jar-asset.jar'), - }); - - // THEN - test.equal(nonZipAsset.isZipArchive, false); - test.equal(zipDirectoryAsset.isZipArchive, true); - test.equal(zipFileAsset.isZipArchive, true); - test.equal(jarFileAsset.isZipArchive, true); - test.done(); - }, - - 'addResourceMetadata can be used to add CFN metadata to resources'(test: Test) { - // GIVEN - const stack = new cdk.Stack(); - stack.node.setContext(cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT, true); - - const location = path.join(__dirname, 'sample-asset-directory'); - const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); - const asset = new Asset(stack, 'MyAsset', { path: location }); - - // WHEN - asset.addResourceMetadata(resource, 'PropName'); - - // THEN - expect(stack).to(haveResource('My::Resource::Type', { - Metadata: { - 'aws:asset:path': 'asset.6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', - 'aws:asset:property': 'PropName', - }, - }, ResourcePart.CompleteDefinition)); - test.done(); - }, - - 'asset metadata is only emitted if ASSET_RESOURCE_METADATA_ENABLED_CONTEXT is defined'(test: Test) { - // GIVEN - const stack = new cdk.Stack(); - - const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); - const asset = new Asset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); - - // WHEN - asset.addResourceMetadata(resource, 'PropName'); - - // THEN - expect(stack).notTo(haveResource('My::Resource::Type', { - Metadata: { - 'aws:asset:path': SAMPLE_ASSET_DIR, - 'aws:asset:property': 'PropName', - }, - }, ResourcePart.CompleteDefinition)); - - test.done(); - }, - - 'staging': { - - 'copy file assets under /${fingerprint}.ext'(test: Test) { - const tempdir = mkdtempSync(); - process.chdir(tempdir); // change current directory to somewhere in /tmp - - // GIVEN - const app = new cdk.App({ outdir: tempdir }); - const stack = new cdk.Stack(app, 'stack'); - - // WHEN - new Asset(stack, 'ZipFile', { - path: path.join(SAMPLE_ASSET_DIR, 'sample-zip-asset.zip'), - }); - - new Asset(stack, 'TextFile', { - path: path.join(SAMPLE_ASSET_DIR, 'sample-asset-file.txt'), - }); - - // THEN - app.synth(); - test.ok(fs.existsSync(tempdir)); - test.ok(fs.existsSync(path.join(tempdir, 'asset.a7a79cdf84b802ea8b198059ff899cffc095a1b9606e919f98e05bf80779756b.zip'))); - test.done(); - }, - - 'copy directory under .assets/fingerprint/**'(test: Test) { - const tempdir = mkdtempSync(); - process.chdir(tempdir); // change current directory to somewhere in /tmp - - // GIVEN - const app = new cdk.App({ outdir: tempdir }); - const stack = new cdk.Stack(app, 'stack'); - - // WHEN - new Asset(stack, 'ZipDirectory', { - path: SAMPLE_ASSET_DIR, - }); - - // THEN - app.synth(); - test.ok(fs.existsSync(tempdir)); - const hash = 'asset.6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2'; - test.ok(fs.existsSync(path.join(tempdir, hash, 'sample-asset-file.txt'))); - test.ok(fs.existsSync(path.join(tempdir, hash, 'sample-jar-asset.jar'))); - fs.readdirSync(tempdir); - test.done(); - }, - - 'staging path is relative if the dir is below the working directory'(test: Test) { - // GIVEN - const tempdir = mkdtempSync(); - process.chdir(tempdir); // change current directory to somewhere in /tmp - - const staging = '.my-awesome-staging-directory'; - const app = new cdk.App({ - outdir: staging, - context: { - [cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT]: 'true', - }, - }); - - const stack = new cdk.Stack(app, 'stack'); - - const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); - const asset = new Asset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); - - // WHEN - asset.addResourceMetadata(resource, 'PropName'); - - const template = SynthUtils.synthesize(stack).template; - test.deepEqual(template.Resources.MyResource.Metadata, { - 'aws:asset:path': 'asset.6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2', - 'aws:asset:property': 'PropName', - }); - test.done(); - }, - - 'if staging is disabled, asset path is absolute'(test: Test) { - // GIVEN - const staging = path.resolve(mkdtempSync()); - const app = new cdk.App({ - outdir: staging, - context: { - [cxapi.DISABLE_ASSET_STAGING_CONTEXT]: 'true', - [cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT]: 'true', - }, - }); - - const stack = new cdk.Stack(app, 'stack'); - - const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); - const asset = new Asset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); - - // WHEN - asset.addResourceMetadata(resource, 'PropName'); - - const template = SynthUtils.synthesize(stack).template; - test.deepEqual(template.Resources.MyResource.Metadata, { - 'aws:asset:path': SAMPLE_ASSET_DIR, - 'aws:asset:property': 'PropName', - }); - test.done(); - }, - - 'cdk metadata points to staged asset'(test: Test) { - // GIVEN - const app = new cdk.App(); - const stack = new cdk.Stack(app, 'stack'); - new Asset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); - - // WHEN - const session = app.synth(); - const artifact = session.getStackByName(stack.stackName); - const metadata = artifact.manifest.metadata || {}; - const md = Object.values(metadata)[0]![0]!.data as cxschema.AssetMetadataEntry; - test.deepEqual(md.path, 'asset.6b84b87243a4a01c592d78e1fd3855c4bfef39328cd0a450cc97e81717fea2a2'); - test.done(); - }, - - }, -}; - -function mkdtempSync() { - return fs.mkdtempSync(path.join(os.tmpdir(), 'test.assets')); -} From 203e16602b4a06f857e0b36bee340522fb0213e7 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Wed, 20 May 2020 13:33:40 +0200 Subject: [PATCH 16/53] jsdoc --- packages/@aws-cdk/aws-lambda/lib/runtime.ts | 2 +- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/runtime.ts b/packages/@aws-cdk/aws-lambda/lib/runtime.ts index b2d1ec4083a7d..0a6949faab5f5 100644 --- a/packages/@aws-cdk/aws-lambda/lib/runtime.ts +++ b/packages/@aws-cdk/aws-lambda/lib/runtime.ts @@ -158,7 +158,7 @@ export class Runtime { /** * The bundling Docker image for this runtime. - * Points to the lambdci/lambda build image for this runtime. + * Points to the lambdci/lambda build image for this runtime. * * @see https://hub.docker.com/r/lambci/lambda/ */ diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 64866e62165bc..4bd3011f78a3f 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -89,7 +89,7 @@ export interface AssetOptions extends assets.CopyOptions { * Bundle the asset by executing a command in a Docker container. * The asset path will be mounted at `/input`. The Docker container is * responsible for putting content at `/output`. The content at `/output` - * will be zipped and used as the asset. + * will be zipped and used as the final asset. * * @default - asset path is zipped as is */ From 21c10a4ae64de169d466d1c687832e0995154ee2 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Wed, 20 May 2020 13:38:36 +0200 Subject: [PATCH 17/53] README --- packages/@aws-cdk/aws-lambda/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index d89cdc7b79155..35930a7501925 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -281,14 +281,16 @@ new lambda.Function(this, 'Function', { ``` Runtimes expose a `bundlingDockerImage` property that points to the [lambci/lambda](https://hub.docker.com/r/lambci/lambda/) build image. -Use `lambda.DockerImage.fromRegistry(image)` to use an existing image or -`lambda.DockerImage.fromAsset(path)` to build a specific image: +Use `assets.BundlingDockerImage.fromRegistry(image)` to use an existing image or +`assets.BundlingDockerImage.fromAsset(path)` to build a specific image: ```ts +import * as assets from '@aws-cdk/aws-s3-assets'; + new lambda.Function(this, 'Function', { code: lambda.Code.fromAsset('/path/to/handler'), { bundling: { - image: lambda.BundlingDockerImage.fromAsset('/path/to/dir/with/DockerFile', { + image: assets.BundlingDockerImage.fromAsset('/path/to/dir/with/DockerFile', { buildArgs: { ARG1: 'value1', }, From 498f5601b6523ce4dcc0bbe7f90c63f9d6a89d44 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Wed, 20 May 2020 15:22:44 +0200 Subject: [PATCH 18/53] fix integ test --- packages/@aws-cdk/aws-lambda/test/integ.bundling.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts index 0c791b6615071..a46f86716041b 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts @@ -20,8 +20,8 @@ class TestStack extends Stack { image: lambda.Runtime.PYTHON_3_6.bundlingDockerImage, command: [ 'bash', '-c', ` - pip install -r requirements.txt -t /bundle && - rsync -r . /bundle + pip install -r requirements.txt -t /output && + rsync -r . /output `, ], }, From c37ed4a57f5ca70272196cae658ecfa41d5c36c6 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Wed, 20 May 2020 22:50:01 +0200 Subject: [PATCH 19/53] reverts --- packages/@aws-cdk/aws-lambda/lib/code.ts | 2 +- packages/@aws-cdk/aws-lambda/lib/function.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/code.ts b/packages/@aws-cdk/aws-lambda/lib/code.ts index f43b4dd076d5e..263e67f415bd4 100644 --- a/packages/@aws-cdk/aws-lambda/lib/code.ts +++ b/packages/@aws-cdk/aws-lambda/lib/code.ts @@ -170,7 +170,7 @@ export class AssetCode extends Code { /** * @param path The path to the asset file or directory. */ - constructor(public readonly path: string, private readonly options: s3_assets.AssetOptions = {}) { + constructor(public readonly path: string, private readonly options: s3_assets.AssetOptions = { }) { super(); } diff --git a/packages/@aws-cdk/aws-lambda/lib/function.ts b/packages/@aws-cdk/aws-lambda/lib/function.ts index 8b4d20a4bc84c..f66a67f07e336 100644 --- a/packages/@aws-cdk/aws-lambda/lib/function.ts +++ b/packages/@aws-cdk/aws-lambda/lib/function.ts @@ -485,9 +485,8 @@ export class Function extends FunctionBase { this.role.addToPolicy(statement); } - this.runtime = props.runtime; const code = props.code.bind(this); - verifyCodeConfig(code, this.runtime); + verifyCodeConfig(code, props.runtime); this.deadLetterQueue = this.buildDeadLetterQueue(props); @@ -523,6 +522,8 @@ export class Function extends FunctionBase { sep: ':', }); + this.runtime = props.runtime; + if (props.layers) { this.addLayers(...props.layers); } @@ -825,4 +826,4 @@ export function verifyCodeConfig(code: CodeConfig, runtime: Runtime) { if (code.inlineCode && !runtime.supportsInlineCode) { throw new Error(`Inline source not allowed for ${runtime.name}`); } -} +} \ No newline at end of file From e91147e5c8f96da34d245bbf865443d72d5ea59b Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Wed, 20 May 2020 22:55:46 +0200 Subject: [PATCH 20/53] experimental and internal --- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 6 +++++- packages/@aws-cdk/aws-s3-assets/lib/bundling.ts | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 4bd3011f78a3f..68b88a5ac9fd0 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -13,6 +13,8 @@ const BUNDLING_OUTPUT_DIR = '/output'; /** * Bundling options + * + * @experimental */ export interface BundlingOptions { /** @@ -92,6 +94,8 @@ export interface AssetOptions extends assets.CopyOptions { * will be zipped and used as the final asset. * * @default - asset path is zipped as is + * + * @experimental */ readonly bundling?: BundlingOptions; } @@ -190,7 +194,7 @@ export class Asset extends cdk.Construct implements assets.IAsset { ...props.bundling.volumes ?? [], ]; - props.bundling.image.run({ + props.bundling.image._run({ command: props.bundling.command, volumes, environment: props.bundling.environment, diff --git a/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts b/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts index 9313f7a05d0ca..90968a30b1913 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts @@ -100,12 +100,14 @@ export class BundlingDockerImage { } /** @param image The Docker image */ - constructor(public readonly image: string) {} + private constructor(public readonly image: string) {} /** * Runs a Docker image + * + * @internal */ - public run(options: DockerRunOptions) { + public _run(options: DockerRunOptions) { const volumes = options.volumes || []; const environment = options.environment || {}; const command = options.command || []; From 248e7f8cc3d010d29b20136fcfd972d048d11d65 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Wed, 20 May 2020 22:59:48 +0200 Subject: [PATCH 21/53] flip order --- packages/@aws-cdk/aws-lambda/test/integ.bundling.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts index a46f86716041b..403a576cda780 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts @@ -19,10 +19,11 @@ class TestStack extends Stack { bundling: { image: lambda.Runtime.PYTHON_3_6.bundlingDockerImage, command: [ - 'bash', '-c', ` - pip install -r requirements.txt -t /output && - rsync -r . /output - `, + 'bash', '-c', [ + 'rsync -r . /output', + 'cd /output', + 'pip install -r requirements.txt -t .', + ].join(' && '), ], }, // Python dependencies do not give a stable hash From 7ada5b756db20225fa58b58ca5925880b836da48 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Thu, 21 May 2020 11:52:16 +0200 Subject: [PATCH 22/53] s3-assets integ, README, JSDoc --- packages/@aws-cdk/aws-lambda/README.md | 8 +++--- packages/@aws-cdk/aws-lambda/lib/runtime.ts | 2 +- .../aws-lambda/test/integ.bundling.ts | 4 +-- packages/@aws-cdk/aws-s3-assets/README.md | 3 +++ .../@aws-cdk/aws-s3-assets/lib/bundling.ts | 4 +-- .../test/alpine-markdown/Dockerfile | 3 +++ .../integ.assets.bundling.lit.expected.json | 16 +++++++++++ .../test/integ.assets.bundling.lit.ts | 27 +++++++++++++++++++ .../test/markdown-asset/index.md | 3 +++ 9 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 packages/@aws-cdk/aws-s3-assets/test/alpine-markdown/Dockerfile create mode 100644 packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json create mode 100644 packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts create mode 100644 packages/@aws-cdk/aws-s3-assets/test/markdown-asset/index.md diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index 35930a7501925..4bcdf4357a3ef 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -257,8 +257,8 @@ retention it seeks to manage. ### Bundling Asset Code When using `lambda.Code.fromAsset(path)` it is possible to bundle the code by running a -command in a Docker container. The asset path will be mounted at `/src`. The Docker -container is responsible for putting content at `/bundle`. The content at `/bundle` +command in a Docker container. The asset path will be mounted at `/input`. The Docker +container is responsible for putting content at `/output`. The content at `/bundle` will be zipped and used as Lambda code. Example with Python: @@ -269,8 +269,8 @@ new lambda.Function(this, 'Function', { image: lambda.Runtime.PYTHON_3_6.bundlingDockerImage, command: [ 'bash', '-c', ` - pip install -r requirements.txt -t /bundle && - rsync -r . /bundle + pip install -r requirements.txt -t /output && + rsync -r . /output `, ], }, diff --git a/packages/@aws-cdk/aws-lambda/lib/runtime.ts b/packages/@aws-cdk/aws-lambda/lib/runtime.ts index 0a6949faab5f5..f77e9967029fe 100644 --- a/packages/@aws-cdk/aws-lambda/lib/runtime.ts +++ b/packages/@aws-cdk/aws-lambda/lib/runtime.ts @@ -158,7 +158,7 @@ export class Runtime { /** * The bundling Docker image for this runtime. - * Points to the lambdci/lambda build image for this runtime. + * Points to the lambci/lambda build image for this runtime. * * @see https://hub.docker.com/r/lambci/lambda/ */ diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts index 403a576cda780..3a54e2bd19d89 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts @@ -4,7 +4,7 @@ import * as lambda from '../lib'; /** * Stack verification steps: - * * aws cloudformation describe-stacks --stack-name cdk-integ-lambda-docker --query Stacks[0].Outputs[0].OutputValue + * * aws cloudformation describe-stacks --stack-name cdk-integ-lambda-bundling --query Stacks[0].Outputs[0].OutputValue * * aws lambda invoke --function-name response.json * * cat response.json * The last command should show '200' @@ -40,5 +40,5 @@ class TestStack extends Stack { } const app = new App(); -new TestStack(app, 'cdk-integ-lambda-docker'); +new TestStack(app, 'cdk-integ-lambda-bundling'); app.synth(); diff --git a/packages/@aws-cdk/aws-s3-assets/README.md b/packages/@aws-cdk/aws-s3-assets/README.md index 86490d0421025..07d3a88bb0208 100644 --- a/packages/@aws-cdk/aws-s3-assets/README.md +++ b/packages/@aws-cdk/aws-s3-assets/README.md @@ -50,6 +50,9 @@ The following examples grants an IAM group read permissions on an asset: [Example of granting read access to an asset](./test/integ.assets.permissions.lit.ts) +The following example uses custom asset bundling to convert a markdown file to html: +[Example of using asset bundling](./test/integ.assets.bundling.lit.ts) + ## How does it work? When an asset is defined in a construct, a construct metadata entry diff --git a/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts b/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts index 90968a30b1913..008dcc9277516 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts @@ -18,7 +18,7 @@ export interface DockerVolume { /** * Docker run options */ -export interface DockerRunOptions { +interface DockerRunOptions { /** * The command to run in the container. * @@ -107,7 +107,7 @@ export class BundlingDockerImage { * * @internal */ - public _run(options: DockerRunOptions) { + public _run(options: DockerRunOptions = {}) { const volumes = options.volumes || []; const environment = options.environment || {}; const command = options.command || []; diff --git a/packages/@aws-cdk/aws-s3-assets/test/alpine-markdown/Dockerfile b/packages/@aws-cdk/aws-s3-assets/test/alpine-markdown/Dockerfile new file mode 100644 index 0000000000000..fa7a67678bae9 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-assets/test/alpine-markdown/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine + +RUN apk add markdown diff --git a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json new file mode 100644 index 0000000000000..764a99e8e10af --- /dev/null +++ b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json @@ -0,0 +1,16 @@ +{ + "Parameters": { + "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8S3Bucket8CD0F73B": { + "Type": "String", + "Description": "S3 bucket for asset \"10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8\"" + }, + "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8S3VersionKeyA9EAF743": { + "Type": "String", + "Description": "S3 key for asset version \"10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8\"" + }, + "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8ArtifactHashBAE492DD": { + "Type": "String", + "Description": "Artifact hash for asset \"10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8\"" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts new file mode 100644 index 0000000000000..f28312bbe7f72 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts @@ -0,0 +1,27 @@ +import { App, Construct, Stack, StackProps } from '@aws-cdk/core'; +import * as path from 'path'; +import * as assets from '../lib'; + +class TestStack extends Stack { + constructor(scope: Construct, id: string, props?: StackProps) { + super(scope, id, props); + + /// !show + new assets.Asset(this, 'BundledAsset', { + path: path.join(__dirname, 'markdown-asset'), // /input and working directory in the container + bundling: { + image: assets.BundlingDockerImage.fromAsset(path.join(__dirname, 'alpine-markdown')), // Build an image + command: [ + 'sh', '-c', ` + markdown index.md > /output/index.html + `, + ], + }, + }); + /// !hide + } +} + +const app = new App(); +new TestStack(app, 'cdk-integ-assets-bundling'); +app.synth(); diff --git a/packages/@aws-cdk/aws-s3-assets/test/markdown-asset/index.md b/packages/@aws-cdk/aws-s3-assets/test/markdown-asset/index.md new file mode 100644 index 0000000000000..64fdacbb595cb --- /dev/null +++ b/packages/@aws-cdk/aws-s3-assets/test/markdown-asset/index.md @@ -0,0 +1,3 @@ +### This is a sample file + +With **markdown** From 4545e8b51a6f497e4da56c1f0c06b7a28f1e3937 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Thu, 21 May 2020 11:54:24 +0200 Subject: [PATCH 23/53] README --- packages/@aws-cdk/aws-lambda/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index 4bcdf4357a3ef..1f44fb3fe40e5 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -258,7 +258,7 @@ retention it seeks to manage. ### Bundling Asset Code When using `lambda.Code.fromAsset(path)` it is possible to bundle the code by running a command in a Docker container. The asset path will be mounted at `/input`. The Docker -container is responsible for putting content at `/output`. The content at `/bundle` +container is responsible for putting content at `/output`. The content at `/output` will be zipped and used as Lambda code. Example with Python: From 902e6001734179bf125cebc2ef3501d4401bc8fb Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Thu, 21 May 2020 13:29:31 +0200 Subject: [PATCH 24/53] JSDoc --- packages/@aws-cdk/aws-s3-assets/lib/bundling.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts b/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts index 008dcc9277516..086a7b1e077f4 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts @@ -61,7 +61,7 @@ export interface DockerBuildOptions { } /** - * A Docker image used for Lambda bundling + * A Docker image used for asset bundling */ export class BundlingDockerImage { /** From 7bdf0680382db5f136452e05f66fc44c22730fe1 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Thu, 21 May 2020 22:28:55 +0200 Subject: [PATCH 25/53] /asset-input and /asset-output --- packages/@aws-cdk/aws-lambda/README.md | 10 +++++----- .../@aws-cdk/aws-lambda/test/integ.bundling.ts | 4 ++-- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 15 ++++++++------- .../@aws-cdk/aws-s3-assets/test/bundling.test.ts | 6 +++--- .../test/integ.assets.bundling.lit.ts | 4 ++-- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index 1f44fb3fe40e5..67a9a6443f99d 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -257,9 +257,9 @@ retention it seeks to manage. ### Bundling Asset Code When using `lambda.Code.fromAsset(path)` it is possible to bundle the code by running a -command in a Docker container. The asset path will be mounted at `/input`. The Docker -container is responsible for putting content at `/output`. The content at `/output` -will be zipped and used as Lambda code. +command in a Docker container. The asset path will be mounted at `/asset-input`. The +Docker container is responsible for putting content at `/asset-output`. The content at +`/asset-output` will be zipped and used as Lambda code. Example with Python: ```ts @@ -269,8 +269,8 @@ new lambda.Function(this, 'Function', { image: lambda.Runtime.PYTHON_3_6.bundlingDockerImage, command: [ 'bash', '-c', ` - pip install -r requirements.txt -t /output && - rsync -r . /output + pip install -r requirements.txt -t /asset-output && + rsync -r . /asset-output `, ], }, diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts index 3a54e2bd19d89..db46e626d8596 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts @@ -20,8 +20,8 @@ class TestStack extends Stack { image: lambda.Runtime.PYTHON_3_6.bundlingDockerImage, command: [ 'bash', '-c', [ - 'rsync -r . /output', - 'cd /output', + 'rsync -r . /asset-output', + 'cd /asset-output', 'pip install -r requirements.txt -t .', ].join(' && '), ], diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 68b88a5ac9fd0..9f3085553493b 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -8,8 +8,8 @@ import * as path from 'path'; import { BundlingDockerImage, DockerVolume } from './bundling'; const ARCHIVE_EXTENSIONS = [ '.zip', '.jar' ]; -const BUNDLING_INPUT_DIR = '/input'; -const BUNDLING_OUTPUT_DIR = '/output'; +const BUNDLING_INPUT_DIR = '/asset-input'; +const BUNDLING_OUTPUT_DIR = '/asset-output'; /** * Bundling options @@ -48,13 +48,13 @@ export interface BundlingOptions { /** * Working directory inside the container. * - * @default /input + * @default /asset-input */ readonly workingDirectory?: string; /** * Bundle directory. Subdirectories named after the asset path will be - * created in this directory and mounted at `/output` in the container. + * created in this directory and mounted at `/asset-output` in the container. * Should be added to your `.gitignore`. * * @default .bundle next to the asset directory @@ -89,9 +89,10 @@ export interface AssetOptions extends assets.CopyOptions { /** * Bundle the asset by executing a command in a Docker container. - * The asset path will be mounted at `/input`. The Docker container is - * responsible for putting content at `/output`. The content at `/output` - * will be zipped and used as the final asset. + * The asset path will be mounted at `/asset-input`. The Docker + * container is responsible for putting content at `/asset-output`. + * The content at `/asset-output` will be zipped and used as the + * final asset. * * @default - asset path is zipped as is * diff --git a/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts b/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts index 48b2e9a1df110..73b71e3c89fbe 100644 --- a/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts @@ -38,11 +38,11 @@ test('bundling with image from registry', () => { expect(spawnSync).toHaveBeenCalledWith('docker', [ 'run', '--rm', - '-v', `${SAMPLE_ASSET_DIR}:/input`, - '-v', expect.stringMatching(new RegExp(`${path.join('.bundle', path.basename(SAMPLE_ASSET_DIR))}:/output$`)), + '-v', `${SAMPLE_ASSET_DIR}:/asset-input`, + '-v', expect.stringMatching(new RegExp(`${path.join('.bundle', path.basename(SAMPLE_ASSET_DIR))}:/asset-output$`)), '--env', 'VAR1=value1', '--env', 'VAR2=value2', - '-w', '/input', + '-w', '/asset-input', image, ...command, ]); diff --git a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts index f28312bbe7f72..7377148903c40 100644 --- a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts +++ b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts @@ -8,12 +8,12 @@ class TestStack extends Stack { /// !show new assets.Asset(this, 'BundledAsset', { - path: path.join(__dirname, 'markdown-asset'), // /input and working directory in the container + path: path.join(__dirname, 'markdown-asset'), // /asset-input and working directory in the container bundling: { image: assets.BundlingDockerImage.fromAsset(path.join(__dirname, 'alpine-markdown')), // Build an image command: [ 'sh', '-c', ` - markdown index.md > /output/index.html + markdown index.md > /asset-output/index.html `, ], }, From 8d999e443f906d6b69cdb76bd2b692221940fc02 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Fri, 22 May 2020 10:41:12 +0200 Subject: [PATCH 26/53] README Co-authored-by: Niranjan Jayakar --- packages/@aws-cdk/aws-lambda/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index 67a9a6443f99d..d5065e2b686ea 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -288,7 +288,7 @@ Use `assets.BundlingDockerImage.fromRegistry(image)` to use an existing image or import * as assets from '@aws-cdk/aws-s3-assets'; new lambda.Function(this, 'Function', { - code: lambda.Code.fromAsset('/path/to/handler'), { + code: lambda.Code.fromAsset('/path/to/handler', { bundling: { image: assets.BundlingDockerImage.fromAsset('/path/to/dir/with/DockerFile', { buildArgs: { From c142de064fb5b31f556599f6719d0051c5dbd2e4 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Fri, 22 May 2020 15:34:09 +0200 Subject: [PATCH 27/53] bundleAssetPath in cwd --- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 14 ++-- .../integ.assets.bundling.lit.expected.json | 74 +++++++++++++++++-- .../test/integ.assets.bundling.lit.ts | 6 +- 3 files changed, 80 insertions(+), 14 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 9f3085553493b..abe31e552d0e6 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -53,11 +53,11 @@ export interface BundlingOptions { readonly workingDirectory?: string; /** - * Bundle directory. Subdirectories named after the asset path will be + * Bundling directory. Subdirectories named after the asset unique id will be * created in this directory and mounted at `/asset-output` in the container. * Should be added to your `.gitignore`. * - * @default .bundle next to the asset directory + * @default ".bundle" under the current working directory */ readonly bundleDirectory?: string; } @@ -172,20 +172,20 @@ export class Asset extends cdk.Construct implements assets.IAsset { // Bundling let bundleAssetPath: string | undefined; if (props.bundling) { - // Create the directory for the bundle next to the asset path - const bundlePath = path.join(path.dirname(props.path), props.bundling.bundleDirectory ?? '.bundle'); + // Create the directory for the bundle in the current working directory + const bundlePath = path.join('./', props.bundling.bundleDirectory ?? '.bundle'); if (!fs.existsSync(bundlePath)) { fs.mkdirSync(bundlePath); } - bundleAssetPath = path.join(bundlePath, path.basename(props.path)); + bundleAssetPath = path.resolve(path.join(bundlePath, this.node.uniqueId)); if (!fs.existsSync(bundleAssetPath)) { fs.mkdirSync(bundleAssetPath); } const volumes = [ { - hostPath: props.path, + hostPath: path.resolve(props.path), containerPath: BUNDLING_INPUT_DIR, }, { @@ -206,7 +206,7 @@ export class Asset extends cdk.Construct implements assets.IAsset { // stage the asset source (conditionally). const staging = new assets.Staging(this, 'Stage', { ...props, - sourcePath: path.resolve(props.path), + sourcePath: path.resolve(bundleAssetPath ?? props.path), }); this.sourceHash = props.sourceHash || staging.sourceHash; diff --git a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json index 764a99e8e10af..9af3702475302 100644 --- a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json +++ b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json @@ -1,16 +1,78 @@ { "Parameters": { - "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8S3Bucket8CD0F73B": { + "AssetParameters4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cbS3Bucket6A787065": { "Type": "String", - "Description": "S3 bucket for asset \"10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8\"" + "Description": "S3 bucket for asset \"4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cb\"" }, - "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8S3VersionKeyA9EAF743": { + "AssetParameters4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cbS3VersionKey61D8476E": { "Type": "String", - "Description": "S3 key for asset version \"10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8\"" + "Description": "S3 key for asset version \"4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cb\"" }, - "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8ArtifactHashBAE492DD": { + "AssetParameters4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cbArtifactHash83AE1BC1": { "Type": "String", - "Description": "Artifact hash for asset \"10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8\"" + "Description": "Artifact hash for asset \"4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cb\"" + } + }, + "Resources": { + "MyUserDC45028B": { + "Type": "AWS::IAM::User" + }, + "MyUserDefaultPolicy7B897426": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Ref": "AssetParameters4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cbS3Bucket6A787065" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Ref": "AssetParameters4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cbS3Bucket6A787065" + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "MyUserDefaultPolicy7B897426", + "Users": [ + { + "Ref": "MyUserDC45028B" + } + ] + } } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts index 7377148903c40..cc78175f8d409 100644 --- a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts +++ b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts @@ -1,3 +1,4 @@ +import * as iam from '@aws-cdk/aws-iam'; import { App, Construct, Stack, StackProps } from '@aws-cdk/core'; import * as path from 'path'; import * as assets from '../lib'; @@ -7,7 +8,7 @@ class TestStack extends Stack { super(scope, id, props); /// !show - new assets.Asset(this, 'BundledAsset', { + const asset = new assets.Asset(this, 'BundledAsset', { path: path.join(__dirname, 'markdown-asset'), // /asset-input and working directory in the container bundling: { image: assets.BundlingDockerImage.fromAsset(path.join(__dirname, 'alpine-markdown')), // Build an image @@ -19,6 +20,9 @@ class TestStack extends Stack { }, }); /// !hide + + const user = new iam.User(this, 'MyUser'); + asset.grantRead(user); } } From 4685d7be79e122f799d0d4f0474975f2c3fe82d0 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Fri, 22 May 2020 15:51:31 +0200 Subject: [PATCH 28/53] try catch around _run --- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index abe31e552d0e6..aa141ed8eb6f0 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -195,12 +195,16 @@ export class Asset extends cdk.Construct implements assets.IAsset { ...props.bundling.volumes ?? [], ]; - props.bundling.image._run({ - command: props.bundling.command, - volumes, - environment: props.bundling.environment, - workingDirectory: props.bundling.workingDirectory ?? BUNDLING_INPUT_DIR, - }); + try { + props.bundling.image._run({ + command: props.bundling.command, + volumes, + environment: props.bundling.environment, + workingDirectory: props.bundling.workingDirectory ?? BUNDLING_INPUT_DIR, + }); + } catch (err) { + throw new Error(`Failed to run bundling Docker image for asset ${id}: ${err}`); + } } // stage the asset source (conditionally). From 453f188da2454f076ac78fe788a67d5f933791ef Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Fri, 22 May 2020 17:44:01 +0200 Subject: [PATCH 29/53] fix bundling test --- packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts b/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts index 73b71e3c89fbe..4995573829407 100644 --- a/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts @@ -10,6 +10,7 @@ jest.mock('child_process'); let stack: Stack; beforeEach(() => { stack = new Stack(); + jest.clearAllMocks(); }); test('bundling with image from registry', () => { @@ -39,7 +40,7 @@ test('bundling with image from registry', () => { expect(spawnSync).toHaveBeenCalledWith('docker', [ 'run', '--rm', '-v', `${SAMPLE_ASSET_DIR}:/asset-input`, - '-v', expect.stringMatching(new RegExp(`${path.join('.bundle', path.basename(SAMPLE_ASSET_DIR))}:/asset-output$`)), + '-v', expect.stringMatching(new RegExp(`${path.join('.bundle', 'Asset')}:/asset-output$`)), '--env', 'VAR1=value1', '--env', 'VAR2=value2', '-w', '/asset-input', @@ -72,13 +73,13 @@ test('bundling with image from asset', () => { }, }); - expect(spawnSync).toHaveBeenCalledWith('docker', [ + expect(spawnSync).toHaveBeenNthCalledWith(1, 'docker', [ 'build', '--build-arg', `TEST_ARG=${testArg}`, dockerPath, ]); - expect(spawnSync).toHaveBeenCalledWith('docker', expect.arrayContaining([ + expect(spawnSync).toHaveBeenNthCalledWith(2, 'docker', expect.arrayContaining([ imageId, ])); }); @@ -119,7 +120,7 @@ test('throws in case of spawnSync error', () => { bundling: { image: assets.BundlingDockerImage.fromRegistry('alpine'), }, - })).toThrow(spawnSyncError); + })).toThrow(spawnSyncError.message); }); test('throws if status is not 0', () => { @@ -137,5 +138,5 @@ test('throws if status is not 0', () => { bundling: { image: assets.BundlingDockerImage.fromRegistry('alpine'), }, - })).toThrow(/^\[Status -1\]/); + })).toThrow(/\[Status -1\]/); }); From c1422d636bcfbb267b80eb5dddc870a40e163af8 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Tue, 26 May 2020 10:19:22 +0200 Subject: [PATCH 30/53] add link to docker run doc --- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index aa141ed8eb6f0..1d34d8f545c76 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -27,6 +27,8 @@ export interface BundlingOptions { * * @example ['npm', 'install'] * + * @see https://docs.docker.com/engine/reference/run/ + * * @default - run the command defined in the image */ readonly command?: string[]; From ce595a35403db66dba91a09cc31563496b0c9db6 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Fri, 5 Jun 2020 22:46:58 +0200 Subject: [PATCH 31/53] start moving to core --- packages/@aws-cdk/assets/lib/api.ts | 2 + packages/@aws-cdk/assets/lib/index.ts | 3 +- packages/@aws-cdk/aws-lambda/lib/runtime.ts | 6 +- .../aws-lambda/test/integ.bundling.ts | 5 +- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 113 ++---------- packages/@aws-cdk/aws-s3-assets/lib/index.ts | 1 - .../aws-s3-assets/test/bundling.test.ts | 142 --------------- .../test/integ.assets.bundling.lit.ts | 4 +- packages/@aws-cdk/core/lib/asset-staging.ts | 171 +++++++++++++++++- packages/@aws-cdk/core/lib/assets.ts | 12 ++ .../{aws-s3-assets => core}/lib/bundling.ts | 3 + packages/@aws-cdk/core/lib/fs/index.ts | 12 +- packages/@aws-cdk/core/lib/index.ts | 1 + packages/@aws-cdk/core/package.json | 2 + packages/@aws-cdk/core/test/test.bundling.ts | 120 ++++++++++++ 15 files changed, 334 insertions(+), 263 deletions(-) delete mode 100644 packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts rename packages/@aws-cdk/{aws-s3-assets => core}/lib/bundling.ts (97%) create mode 100644 packages/@aws-cdk/core/test/test.bundling.ts diff --git a/packages/@aws-cdk/assets/lib/api.ts b/packages/@aws-cdk/assets/lib/api.ts index 75966e57d5af8..a575c92c293a9 100644 --- a/packages/@aws-cdk/assets/lib/api.ts +++ b/packages/@aws-cdk/assets/lib/api.ts @@ -1,5 +1,7 @@ /** * Common interface for all assets. + * + * @deprecated use `core.IAsset` */ export interface IAsset { /** diff --git a/packages/@aws-cdk/assets/lib/index.ts b/packages/@aws-cdk/assets/lib/index.ts index c651e06cc2ac1..0ad070b4d52e6 100644 --- a/packages/@aws-cdk/assets/lib/index.ts +++ b/packages/@aws-cdk/assets/lib/index.ts @@ -1,4 +1,5 @@ export * from './api'; export * from './fs/follow-mode'; export * from './fs/options'; -export * from './staging'; \ No newline at end of file +export * from './staging'; +export * from './compat'; diff --git a/packages/@aws-cdk/aws-lambda/lib/runtime.ts b/packages/@aws-cdk/aws-lambda/lib/runtime.ts index f77e9967029fe..25f36b6a4f53f 100644 --- a/packages/@aws-cdk/aws-lambda/lib/runtime.ts +++ b/packages/@aws-cdk/aws-lambda/lib/runtime.ts @@ -1,4 +1,4 @@ -import * as assets from '@aws-cdk/aws-s3-assets'; +import { BundlingDockerImage } from '@aws-cdk/core'; export interface LambdaRuntimeProps { /** @@ -162,13 +162,13 @@ export class Runtime { * * @see https://hub.docker.com/r/lambci/lambda/ */ - public readonly bundlingDockerImage: assets.BundlingDockerImage; + public readonly bundlingDockerImage: BundlingDockerImage; constructor(name: string, family?: RuntimeFamily, props: LambdaRuntimeProps = { }) { this.name = name; this.supportsInlineCode = !!props.supportsInlineCode; this.family = family; - this.bundlingDockerImage = assets.BundlingDockerImage.fromRegistry(`lambci/lambda:build-${name}`); + this.bundlingDockerImage = BundlingDockerImage.fromRegistry(`lambci/lambda:build-${name}`); Runtime.ALL.push(this); } diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts index db46e626d8596..4eebbd8f23582 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts @@ -1,4 +1,4 @@ -import { App, CfnOutput, Construct, FileSystem, Stack, StackProps } from '@aws-cdk/core'; +import { App, CfnOutput, Construct, HashCalculation, Stack, StackProps } from '@aws-cdk/core'; import * as path from 'path'; import * as lambda from '../lib'; @@ -25,9 +25,8 @@ class TestStack extends Stack { 'pip install -r requirements.txt -t .', ].join(' && '), ], + hashCalculation: HashCalculation.SOURCE, }, - // Python dependencies do not give a stable hash - sourceHash: FileSystem.fingerprint(assetPath), }), runtime: lambda.Runtime.PYTHON_3_6, handler: 'index.handler', diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 1d34d8f545c76..5560864e702d2 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -5,64 +5,8 @@ import * as cdk from '@aws-cdk/core'; import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs'; import * as path from 'path'; -import { BundlingDockerImage, DockerVolume } from './bundling'; const ARCHIVE_EXTENSIONS = [ '.zip', '.jar' ]; -const BUNDLING_INPUT_DIR = '/asset-input'; -const BUNDLING_OUTPUT_DIR = '/asset-output'; - -/** - * Bundling options - * - * @experimental - */ -export interface BundlingOptions { - /** - * The Docker image where the command will run. - */ - readonly image: BundlingDockerImage; - - /** - * The command to run in the container. - * - * @example ['npm', 'install'] - * - * @see https://docs.docker.com/engine/reference/run/ - * - * @default - run the command defined in the image - */ - readonly command?: string[]; - - /** - * Additional Docker volumes to mount. - * - * @default - no additional volumes are mounted - */ - readonly volumes?: DockerVolume[]; - - /** - * The environment variables to pass to the container. - * - * @default - no environment variables. - */ - readonly environment?: { [key: string]: string; }; - - /** - * Working directory inside the container. - * - * @default /asset-input - */ - readonly workingDirectory?: string; - - /** - * Bundling directory. Subdirectories named after the asset unique id will be - * created in this directory and mounted at `/asset-output` in the container. - * Should be added to your `.gitignore`. - * - * @default ".bundle" under the current working directory - */ - readonly bundleDirectory?: string; -} export interface AssetOptions extends assets.CopyOptions { /** @@ -96,11 +40,12 @@ export interface AssetOptions extends assets.CopyOptions { * The content at `/asset-output` will be zipped and used as the * final asset. * - * @default - asset path is zipped as is + * @default - uploaded as-is to S3 if the asset is a regular file or a .zip file, + * archived into a .zip file and uploaded to S3 otherwise * * @experimental */ - readonly bundling?: BundlingOptions; + readonly bundling?: cdk.BundlingOptions; } export interface AssetProps extends AssetOptions { @@ -118,7 +63,7 @@ export interface AssetProps extends AssetOptions { * An asset represents a local file or directory, which is automatically uploaded to S3 * and then can be referenced within a CDK application. */ -export class Asset extends cdk.Construct implements assets.IAsset { +export class Asset extends cdk.Construct implements cdk.IAsset { /** * Attribute that represents the name of the bucket this asset exists in. */ @@ -171,51 +116,15 @@ export class Asset extends cdk.Construct implements assets.IAsset { constructor(scope: cdk.Construct, id: string, props: AssetProps) { super(scope, id); - // Bundling - let bundleAssetPath: string | undefined; - if (props.bundling) { - // Create the directory for the bundle in the current working directory - const bundlePath = path.join('./', props.bundling.bundleDirectory ?? '.bundle'); - if (!fs.existsSync(bundlePath)) { - fs.mkdirSync(bundlePath); - } - - bundleAssetPath = path.resolve(path.join(bundlePath, this.node.uniqueId)); - if (!fs.existsSync(bundleAssetPath)) { - fs.mkdirSync(bundleAssetPath); - } - - const volumes = [ - { - hostPath: path.resolve(props.path), - containerPath: BUNDLING_INPUT_DIR, - }, - { - hostPath: bundleAssetPath, - containerPath: BUNDLING_OUTPUT_DIR, - }, - ...props.bundling.volumes ?? [], - ]; - - try { - props.bundling.image._run({ - command: props.bundling.command, - volumes, - environment: props.bundling.environment, - workingDirectory: props.bundling.workingDirectory ?? BUNDLING_INPUT_DIR, - }); - } catch (err) { - throw new Error(`Failed to run bundling Docker image for asset ${id}: ${err}`); - } - } - // stage the asset source (conditionally). - const staging = new assets.Staging(this, 'Stage', { - ...props, - sourcePath: path.resolve(bundleAssetPath ?? props.path), + const staging = new cdk.AssetStaging(this, 'Stage', { + sourcePath: path.resolve(props.path), + exclude: props.exclude, + follow: assets.toSymlinkFollow(props.follow), + bundling: props.bundling, }); - this.sourceHash = props.sourceHash || staging.sourceHash; + this.sourceHash = props.sourceHash ?? staging.sourceHash; this.assetPath = staging.stagedPath; @@ -242,7 +151,7 @@ export class Asset extends cdk.Construct implements assets.IAsset { this.bucket = s3.Bucket.fromBucketName(this, 'AssetBucket', this.s3BucketName); - for (const reader of (props.readers || [])) { + for (const reader of (props.readers ?? [])) { this.grantRead(reader); } } diff --git a/packages/@aws-cdk/aws-s3-assets/lib/index.ts b/packages/@aws-cdk/aws-s3-assets/lib/index.ts index 3ea08e61cb5b9..ea2719dd83bd3 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/index.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/index.ts @@ -1,2 +1 @@ export * from './asset'; -export * from './bundling'; diff --git a/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts b/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts deleted file mode 100644 index 4995573829407..0000000000000 --- a/packages/@aws-cdk/aws-s3-assets/test/bundling.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { Stack } from '@aws-cdk/core'; -import { spawnSync } from 'child_process'; -import * as path from 'path'; -import * as assets from '../lib'; - -const SAMPLE_ASSET_DIR = path.join(__dirname, 'sample-asset-directory'); - -jest.mock('child_process'); - -let stack: Stack; -beforeEach(() => { - stack = new Stack(); - jest.clearAllMocks(); -}); - -test('bundling with image from registry', () => { - (spawnSync as jest.Mock).mockImplementation((): ReturnType => ({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - })); - - const command = ['this', 'is', 'a', 'build', 'command']; - const image = 'alpine'; - new assets.Asset(stack, 'Asset', { - path: SAMPLE_ASSET_DIR, - bundling: { - image: assets.BundlingDockerImage.fromRegistry(image), - environment: { - VAR1: 'value1', - VAR2: 'value2', - }, - command, - }, - }); - - expect(spawnSync).toHaveBeenCalledWith('docker', [ - 'run', '--rm', - '-v', `${SAMPLE_ASSET_DIR}:/asset-input`, - '-v', expect.stringMatching(new RegExp(`${path.join('.bundle', 'Asset')}:/asset-output$`)), - '--env', 'VAR1=value1', - '--env', 'VAR2=value2', - '-w', '/asset-input', - image, - ...command, - ]); -}); - -test('bundling with image from asset', () => { - const imageId = 'abcdef123456'; - (spawnSync as jest.Mock).mockImplementation((): ReturnType => ({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from(`Successfully built ${imageId}`), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - })); - - const dockerPath = 'docker-path'; - const testArg = 'cdk-test'; - new assets.Asset(stack, 'Asset', { - path: SAMPLE_ASSET_DIR, - bundling: { - image: assets.BundlingDockerImage.fromAsset(dockerPath, { - buildArgs: { - TEST_ARG: testArg, - }, - }), - }, - }); - - expect(spawnSync).toHaveBeenNthCalledWith(1, 'docker', [ - 'build', - '--build-arg', `TEST_ARG=${testArg}`, - dockerPath, - ]); - - expect(spawnSync).toHaveBeenNthCalledWith(2, 'docker', expect.arrayContaining([ - imageId, - ])); -}); - -test('throws if image id cannot be extracted from build output', () => { - (spawnSync as jest.Mock).mockImplementation((): ReturnType => ({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - })); - - const dockerPath = 'docker-path'; - expect(() => new assets.Asset(stack, 'Asset', { - path: SAMPLE_ASSET_DIR, - bundling: { - image: assets.BundlingDockerImage.fromAsset(dockerPath), - }, - })).toThrow(/Failed to extract image ID from Docker build output/); -}); - -test('throws in case of spawnSync error', () => { - const spawnSyncError = new Error('UnknownError'); - (spawnSync as jest.Mock).mockImplementation((): ReturnType => ({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - error: spawnSyncError, - })); - - expect(() => new assets.Asset(stack, 'Asset', { - path: SAMPLE_ASSET_DIR, - bundling: { - image: assets.BundlingDockerImage.fromRegistry('alpine'), - }, - })).toThrow(spawnSyncError.message); -}); - -test('throws if status is not 0', () => { - (spawnSync as jest.Mock).mockImplementation((): ReturnType => ({ - status: -1, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - })); - - expect(() => new assets.Asset(stack, 'Asset', { - path: SAMPLE_ASSET_DIR, - bundling: { - image: assets.BundlingDockerImage.fromRegistry('alpine'), - }, - })).toThrow(/\[Status -1\]/); -}); diff --git a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts index cc78175f8d409..b1b144f2de275 100644 --- a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts +++ b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.ts @@ -1,5 +1,5 @@ import * as iam from '@aws-cdk/aws-iam'; -import { App, Construct, Stack, StackProps } from '@aws-cdk/core'; +import { App, BundlingDockerImage, Construct, Stack, StackProps } from '@aws-cdk/core'; import * as path from 'path'; import * as assets from '../lib'; @@ -11,7 +11,7 @@ class TestStack extends Stack { const asset = new assets.Asset(this, 'BundledAsset', { path: path.join(__dirname, 'markdown-asset'), // /asset-input and working directory in the container bundling: { - image: assets.BundlingDockerImage.fromAsset(path.join(__dirname, 'alpine-markdown')), // Build an image + image: BundlingDockerImage.fromAsset(path.join(__dirname, 'alpine-markdown')), // Build an image command: [ 'sh', '-c', ` markdown index.md > /asset-output/index.html diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 0fb9dc3da8265..a3ccde7dc5a12 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -1,6 +1,8 @@ import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs'; +import * as os from 'os'; import * as path from 'path'; +import { BUNDLING_INPUT_DIR, BUNDLING_OUTPUT_DIR, BundlingDockerImage, DockerVolume } from './bundling'; import { Construct, ISynthesisSession } from './construct-compat'; import { FileSystem, FingerprintOptions } from './fs'; @@ -12,6 +14,84 @@ export interface AssetStagingProps extends FingerprintOptions { * The source file or directory to copy from. */ readonly sourcePath: string; + + /** + * Bundle the asset by executing a command in a Docker container. + * The asset path will be mounted at `/asset-input`. The Docker + * container is responsible for putting content at `/asset-output`. + * The content at `/asset-output` will used as the final asset. + * + * @default - source is copied to staging directory + * + * @experimental + */ + readonly bundling?: BundlingOptions; +} + +/** + * Bundling options + * + * @experimental + */ +export interface BundlingOptions { + /** + * The Docker image where the command will run. + */ + readonly image: BundlingDockerImage; + + /** + * The command to run in the container. + * + * @example ['npm', 'install'] + * + * @see https://docs.docker.com/engine/reference/run/ + * + * @default - run the command defined in the image + */ + readonly command?: string[]; + + /** + * Additional Docker volumes to mount. + * + * @default - no additional volumes are mounted + */ + readonly volumes?: DockerVolume[]; + + /** + * The environment variables to pass to the container. + * + * @default - no environment variables. + */ + readonly environment?: { [key: string]: string; }; + + /** + * Working directory inside the container. + * + * @default /asset-input + */ + readonly workingDirectory?: string; + + /** + * How to calculate the source hash for this asset. + * + * @default BUNDLE + */ + readonly hashCalculation?: HashCalculation; +} + +/** + * Source hash calculation + */ +export enum HashCalculation { + /** + * Based on the content of the source path + */ + SOURCE = 'source', + + /** + * Based on the content of the bundled path + */ + BUNDLE = 'bundle', } /** @@ -33,7 +113,6 @@ export interface AssetStagingProps extends FingerprintOptions { * means that only if content was changed, copy will happen. */ export class AssetStaging extends Construct { - /** * The path to the asset (stringinfied token). * @@ -56,35 +135,107 @@ export class AssetStaging extends Construct { private readonly relativePath?: string; + private readonly bundleDir?: string; + constructor(scope: Construct, id: string, props: AssetStagingProps) { super(scope, id); this.sourcePath = props.sourcePath; this.fingerprintOptions = props; - this.sourceHash = FileSystem.fingerprint(this.sourcePath, props); const stagingDisabled = this.node.tryGetContext(cxapi.DISABLE_ASSET_STAGING_CONTEXT); - if (stagingDisabled) { - this.stagedPath = this.sourcePath; + + if (props.bundling) { + // Create temporary directory for bundling + this.bundleDir = fs.mkdtempSync(path.resolve(path.join(os.tmpdir(), 'cdk-asset-bundle-'))); + + // Always mount input and output dir + const volumes = [ + { + hostPath: this.sourcePath, + containerPath: BUNDLING_INPUT_DIR, + }, + { + hostPath: this.bundleDir, + containerPath: BUNDLING_OUTPUT_DIR, + }, + ...props.bundling.volumes ?? [], + ]; + + try { + props.bundling.image._run({ + command: props.bundling.command, + volumes, + environment: props.bundling.environment, + workingDirectory: props.bundling.workingDirectory ?? BUNDLING_INPUT_DIR, + }); + } catch (err) { + throw new Error(`Failed to run bundling Docker image for asset ${this.node.id}: ${err}`); + } + + if (FileSystem.isEmpty(this.bundleDir)) { + throw new Error(`Bundling did not produce any output. Check that your container writes content to ${BUNDLING_OUTPUT_DIR}.`); + } + + const hashCalculation = props.bundling.hashCalculation ?? HashCalculation.BUNDLE; + if (hashCalculation === HashCalculation.SOURCE) { + this.sourceHash = this.fingerprint(this.sourcePath); + } else if (hashCalculation === HashCalculation.BUNDLE) { + this.sourceHash = this.fingerprint(this.bundleDir); + } else { + throw new Error('Unknown source hash calculation.'); + } + if (stagingDisabled) { + this.stagedPath = this.bundleDir; + } else { + this.relativePath = `asset.${this.fingerprint(this.bundleDir)}`; // always bundle based + this.stagedPath = this.relativePath; // always relative to outdir + } } else { - this.relativePath = 'asset.' + this.sourceHash + path.extname(this.sourcePath); - this.stagedPath = this.relativePath; // always relative to outdir + this.sourceHash = this.fingerprint(this.sourcePath); + if (stagingDisabled) { + this.stagedPath = this.sourcePath; + } else { + this.relativePath = `asset.${this.sourceHash}${path.extname(this.sourcePath)}`; + this.stagedPath = this.relativePath; // always relative to outdir + } } } protected synthesize(session: ISynthesisSession) { + // Staging is disabled if (!this.relativePath) { return; } const targetPath = path.join(session.assembly.outdir, this.relativePath); - // asset already staged + // Already staged. This also works with bundling because relative + // path is always bundle based when bundling if (fs.existsSync(targetPath)) { return; } - // copy file/directory to staging directory + // Asset has been bundled + if (this.bundleDir) { + // Try to rename bundling directory to staging directory + try { + fs.renameSync(this.bundleDir, targetPath); + return; + } catch (err) { + // /tmp and cdk.out could be mounted across different mount points + // in this case we will fallback to copying. This can happen in WSL. + if (err.code === 'EXDEV') { + fs.mkdirSync(targetPath); + FileSystem.copyDirectory(this.bundleDir, targetPath, this.fingerprintOptions); + return; + } + + throw err; + } + } + + // Copy file/directory to staging directory const stat = fs.statSync(this.sourcePath); if (stat.isFile()) { fs.copyFileSync(this.sourcePath, targetPath); @@ -95,4 +246,8 @@ export class AssetStaging extends Construct { throw new Error(`Unknown file type: ${this.sourcePath}`); } } + + private fingerprint(fileOrDirectory: string) { + return FileSystem.fingerprint(fileOrDirectory, this.fingerprintOptions); + } } diff --git a/packages/@aws-cdk/core/lib/assets.ts b/packages/@aws-cdk/core/lib/assets.ts index 8c59e576b588c..141251062a786 100644 --- a/packages/@aws-cdk/core/lib/assets.ts +++ b/packages/@aws-cdk/core/lib/assets.ts @@ -1,3 +1,15 @@ +/** + * Common interface for all assets. + */ +export interface IAsset { + /** + * A hash of the source of this asset, which is available at construction time. As this is a plain + * string, it can be used in construct IDs in order to enforce creation of a new resource when + * the content hash has changed. + */ + readonly sourceHash: string; +} + /** * Represents the source for a file asset. */ diff --git a/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts similarity index 97% rename from packages/@aws-cdk/aws-s3-assets/lib/bundling.ts rename to packages/@aws-cdk/core/lib/bundling.ts index 086a7b1e077f4..5da07fe481787 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -1,5 +1,8 @@ import { spawnSync } from 'child_process'; +export const BUNDLING_INPUT_DIR = '/asset-input'; +export const BUNDLING_OUTPUT_DIR = '/asset-output'; + /** * A Docker volume */ diff --git a/packages/@aws-cdk/core/lib/fs/index.ts b/packages/@aws-cdk/core/lib/fs/index.ts index ac7f3c9d0f8da..68f04a6727b08 100644 --- a/packages/@aws-cdk/core/lib/fs/index.ts +++ b/packages/@aws-cdk/core/lib/fs/index.ts @@ -1,3 +1,4 @@ +import * as fs from 'fs'; import { copyDirectory } from './copy'; import { fingerprint } from './fingerprint'; import { CopyOptions, FingerprintOptions } from './options'; @@ -33,4 +34,13 @@ export class FileSystem { public static fingerprint(fileOrDirectory: string, options: FingerprintOptions = { }) { return fingerprint(fileOrDirectory, options); } -} \ No newline at end of file + + /** + * Checks whether a directory is empty + * + * @param dir The direcotry to check + */ + public static isEmpty(dir: string): boolean { + return fs.readdirSync(dir).length === 0; + } +} diff --git a/packages/@aws-cdk/core/lib/index.ts b/packages/@aws-cdk/core/lib/index.ts index 4e55122d5616f..c0f32f7c5b6e0 100644 --- a/packages/@aws-cdk/core/lib/index.ts +++ b/packages/@aws-cdk/core/lib/index.ts @@ -47,6 +47,7 @@ export * from './assets'; export * from './tree'; export * from './asset-staging'; +export * from './bundling'; export * from './fs'; export * from './custom-resource'; diff --git a/packages/@aws-cdk/core/package.json b/packages/@aws-cdk/core/package.json index a654253f2d938..3e5b22b6b8a74 100644 --- a/packages/@aws-cdk/core/package.json +++ b/packages/@aws-cdk/core/package.json @@ -155,12 +155,14 @@ "@types/node": "^10.17.21", "@types/nodeunit": "^0.0.31", "@types/minimatch": "^3.0.3", + "@types/sinon": "^9.0.4", "cdk-build-tools": "0.0.0", "cfn2ts": "0.0.0", "fast-check": "^1.24.2", "lodash": "^4.17.15", "nodeunit": "^0.11.3", "pkglint": "0.0.0", + "sinon": "^9.0.2", "ts-mock-imports": "^1.3.0" }, "dependencies": { diff --git a/packages/@aws-cdk/core/test/test.bundling.ts b/packages/@aws-cdk/core/test/test.bundling.ts new file mode 100644 index 0000000000000..658aa99901bb6 --- /dev/null +++ b/packages/@aws-cdk/core/test/test.bundling.ts @@ -0,0 +1,120 @@ +import * as child_process from 'child_process'; +import { Test } from 'nodeunit'; +import * as sinon from 'sinon'; +import { BundlingDockerImage } from '../lib'; + +export = { + 'tearDown'(callback: any) { + sinon.restore(); + callback(); + }, + + 'bundling with image from registry'(test: Test) { + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + + const image = BundlingDockerImage.fromRegistry('alpine'); + image._run({ + command: ['cool', 'command'], + environment: { + VAR1: 'value1', + VAR2: 'value2', + }, + volumes: [{ hostPath: '/host-path', containerPath: '/container-path' }], + workingDirectory: '/working-directory', + }); + + test.ok(spawnSyncStub.calledWith('docker', [ + 'run', '--rm', + '-v', '/host-path:/container-path', + '--env', 'VAR1=value1', + '--env', 'VAR2=value2', + '-w', '/working-directory', + 'alpine', + 'cool', 'command', + ])); + test.done(); + }, + + 'bundling with image from asset'(test: Test) { + const imageId = 'abcdef123456'; + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from(`Successfully built ${imageId}`), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + + const image = BundlingDockerImage.fromAsset('docker-path', { + buildArgs: { + TEST_ARG: 'cdk-test', + }, + }); + image._run(); + + test.ok(spawnSyncStub.firstCall.calledWith('docker', [ + 'build', + '--build-arg', 'TEST_ARG=cdk-test', + 'docker-path', + ])); + + test.ok(spawnSyncStub.secondCall.calledWith('docker', [ + 'run', '--rm', + imageId, + ])); + test.done(); + }, + + 'throws if image id cannot be extracted from build output'(test: Test) { + sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + + test.throws(() => BundlingDockerImage.fromAsset('docker-path'), /Failed to extract image ID from Docker build output/); + test.done(); + }, + + 'throws in case of spawnSync error'(test: Test) { + sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + error: new Error('UnknownError'), + }); + + const image = BundlingDockerImage.fromRegistry('alpine'); + test.throws(() => image._run(), /UnknownError/); + test.done(); + }, + + 'throws if status is not 0'(test: Test) { + sinon.stub(child_process, 'spawnSync').returns({ + status: -1, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + + const image = BundlingDockerImage.fromRegistry('alpine'); + test.throws(() => image._run(), /\[Status -1\]/); + test.done(); + }, +}; From e0112eaf1485a798242dfee389d02f76cc813c83 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Fri, 5 Jun 2020 23:06:44 +0200 Subject: [PATCH 32/53] revert .bundle --- packages/@aws-cdk/aws-lambda/.gitignore | 2 -- packages/@aws-cdk/aws-s3-assets/.gitignore | 2 -- 2 files changed, 4 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/.gitignore b/packages/@aws-cdk/aws-lambda/.gitignore index 7b043ef8be063..2d2f100c9395d 100644 --- a/packages/@aws-cdk/aws-lambda/.gitignore +++ b/packages/@aws-cdk/aws-lambda/.gitignore @@ -15,5 +15,3 @@ nyc.config.js .LAST_PACKAGE *.snk !.eslintrc.js - -.bundle diff --git a/packages/@aws-cdk/aws-s3-assets/.gitignore b/packages/@aws-cdk/aws-s3-assets/.gitignore index 9612a7e9c2696..743b39099999a 100644 --- a/packages/@aws-cdk/aws-s3-assets/.gitignore +++ b/packages/@aws-cdk/aws-s3-assets/.gitignore @@ -16,5 +16,3 @@ nyc.config.js *.snk !.eslintrc.js !jest.config.js - -.bundle From 587bc8235fbb2d1b1a33d8a22e2fe03495d47866 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Sat, 6 Jun 2020 13:32:05 +0200 Subject: [PATCH 33/53] AssetHashCalculation --- packages/@aws-cdk/aws-lambda/test/integ.bundling.ts | 4 ++-- packages/@aws-cdk/core/lib/asset-staging.ts | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts index 4eebbd8f23582..19079342b65a2 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts @@ -1,4 +1,4 @@ -import { App, CfnOutput, Construct, HashCalculation, Stack, StackProps } from '@aws-cdk/core'; +import { App, AssetHashCalculation, CfnOutput, Construct, Stack, StackProps } from '@aws-cdk/core'; import * as path from 'path'; import * as lambda from '../lib'; @@ -25,7 +25,7 @@ class TestStack extends Stack { 'pip install -r requirements.txt -t .', ].join(' && '), ], - hashCalculation: HashCalculation.SOURCE, + hashCalculation: AssetHashCalculation.SOURCE, }, }), runtime: lambda.Runtime.PYTHON_3_6, diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index a3ccde7dc5a12..34f420c90becb 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -76,13 +76,13 @@ export interface BundlingOptions { * * @default BUNDLE */ - readonly hashCalculation?: HashCalculation; + readonly hashCalculation?: AssetHashCalculation; } /** * Source hash calculation */ -export enum HashCalculation { +export enum AssetHashCalculation { /** * Based on the content of the source path */ @@ -177,10 +177,10 @@ export class AssetStaging extends Construct { throw new Error(`Bundling did not produce any output. Check that your container writes content to ${BUNDLING_OUTPUT_DIR}.`); } - const hashCalculation = props.bundling.hashCalculation ?? HashCalculation.BUNDLE; - if (hashCalculation === HashCalculation.SOURCE) { + const hashCalculation = props.bundling.hashCalculation ?? AssetHashCalculation.BUNDLE; + if (hashCalculation === AssetHashCalculation.SOURCE) { this.sourceHash = this.fingerprint(this.sourcePath); - } else if (hashCalculation === HashCalculation.BUNDLE) { + } else if (hashCalculation === AssetHashCalculation.BUNDLE) { this.sourceHash = this.fingerprint(this.bundleDir); } else { throw new Error('Unknown source hash calculation.'); From ea7a48f0c5a1d8bd0e2f94fa62d4d8493b807d64 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 8 Jun 2020 10:14:49 +0200 Subject: [PATCH 34/53] lambda README --- packages/@aws-cdk/aws-lambda/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/README.md b/packages/@aws-cdk/aws-lambda/README.md index ca9610dcd0f5e..01b211d16e142 100644 --- a/packages/@aws-cdk/aws-lambda/README.md +++ b/packages/@aws-cdk/aws-lambda/README.md @@ -63,7 +63,7 @@ const fn = new lambda.Function(this, 'MyFunction', { runtime: lambda.Runtime.NODEJS_10_X, handler: 'index.handler', code: lambda.Code.fromAsset(path.join(__dirname, 'lambda-handler')), - + fn.role // the Role ``` @@ -314,16 +314,16 @@ new lambda.Function(this, 'Function', { ``` Runtimes expose a `bundlingDockerImage` property that points to the [lambci/lambda](https://hub.docker.com/r/lambci/lambda/) build image. -Use `assets.BundlingDockerImage.fromRegistry(image)` to use an existing image or -`assets.BundlingDockerImage.fromAsset(path)` to build a specific image: +Use `cdk.BundlingDockerImage.fromRegistry(image)` to use an existing image or +`cdk.BundlingDockerImage.fromAsset(path)` to build a specific image: ```ts -import * as assets from '@aws-cdk/aws-s3-assets'; +import * as cdk from '@aws-cdk/core'; new lambda.Function(this, 'Function', { code: lambda.Code.fromAsset('/path/to/handler', { bundling: { - image: assets.BundlingDockerImage.fromAsset('/path/to/dir/with/DockerFile', { + image: cdk.BundlingDockerImage.fromAsset('/path/to/dir/with/DockerFile', { buildArgs: { ARG1: 'value1', }, From 706cb48d2a90e7c4ba09ecabdd1382ed4038e7ab Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 8 Jun 2020 17:16:05 +0200 Subject: [PATCH 35/53] PR feedback --- .../aws-lambda/test/integ.bundling.ts | 4 +- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 29 ++++- packages/@aws-cdk/core/lib/asset-staging.ts | 73 ++++--------- packages/@aws-cdk/core/lib/bundling.ts | 102 +++++++++++++----- 4 files changed, 121 insertions(+), 87 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts index 19079342b65a2..c6add465805ba 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts @@ -1,4 +1,4 @@ -import { App, AssetHashCalculation, CfnOutput, Construct, Stack, StackProps } from '@aws-cdk/core'; +import { App, AssetHashType, CfnOutput, Construct, Stack, StackProps } from '@aws-cdk/core'; import * as path from 'path'; import * as lambda from '../lib'; @@ -25,8 +25,8 @@ class TestStack extends Stack { 'pip install -r requirements.txt -t .', ].join(' && '), ], - hashCalculation: AssetHashCalculation.SOURCE, }, + assetHashType: AssetHashType.SOURCE, }), runtime: lambda.Runtime.PYTHON_3_6, handler: 'index.handler', diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 5560864e702d2..d36ae4646ac74 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -5,6 +5,7 @@ import * as cdk from '@aws-cdk/core'; import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs'; import * as path from 'path'; +import { AssetHashType } from '@aws-cdk/core'; const ARCHIVE_EXTENSIONS = [ '.zip', '.jar' ]; @@ -17,6 +18,25 @@ export interface AssetOptions extends assets.CopyOptions { */ readonly readers?: iam.IGrantable[]; + /** + * Specify a custom hash for this asset. If `assetHashType` is set it must + * be set to `AssetHashType.CUSTOM`. + * + * @default - based on `assetHashType` + */ + readonly assetHash?: string; + + /** + * Specifies the type of hash to calculate for this asset. + * + * If `assetHash` is configured, this option must be `undefined` or + * `AssetHashType.CUSTOM`. + * + * @default - the default is `AssetHashType.SOURCE`, but if `assetHash` is + * explicitly specified this value defaults to `AssetHashType.CUSTOM`. + */ + readonly assetHashType?: cdk.AssetHashType; + /** * Custom source hash to use when identifying the specific version of the asset. * @@ -29,7 +49,7 @@ export interface AssetOptions extends assets.CopyOptions { * @default - automatically calculate source hash based on the contents * of the source file or directory. * - * @experimental + * @deprecated see `assetHash` and `assetHashType` */ readonly sourceHash?: string; @@ -116,15 +136,20 @@ export class Asset extends cdk.Construct implements cdk.IAsset { constructor(scope: cdk.Construct, id: string, props: AssetProps) { super(scope, id); + if ((props.assetHash || props.sourceHash) && props.assetHashType && props.assetHashType !== AssetHashType.CUSTOM) { + throw new Error(`Cannot specify \`${props.assetHashType}\` for \`assetHashType\` when \`assetHash\` is specified. Use \`CUSTOM\` or leave \`undefined\`.`); + } + // stage the asset source (conditionally). const staging = new cdk.AssetStaging(this, 'Stage', { sourcePath: path.resolve(props.path), exclude: props.exclude, follow: assets.toSymlinkFollow(props.follow), bundling: props.bundling, + assetHashType: props.assetHashType, }); - this.sourceHash = props.sourceHash ?? staging.sourceHash; + this.sourceHash = props.assetHash ?? props.sourceHash ?? staging.sourceHash; this.assetPath = staging.stagedPath; diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 34f420c90becb..99b22b651b241 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -2,7 +2,7 @@ import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; -import { BUNDLING_INPUT_DIR, BUNDLING_OUTPUT_DIR, BundlingDockerImage, DockerVolume } from './bundling'; +import { BUNDLING_INPUT_DIR, BUNDLING_OUTPUT_DIR, BundlingOptions } from './bundling'; import { Construct, ISynthesisSession } from './construct-compat'; import { FileSystem, FingerprintOptions } from './fs'; @@ -26,63 +26,19 @@ export interface AssetStagingProps extends FingerprintOptions { * @experimental */ readonly bundling?: BundlingOptions; -} - -/** - * Bundling options - * - * @experimental - */ -export interface BundlingOptions { - /** - * The Docker image where the command will run. - */ - readonly image: BundlingDockerImage; - - /** - * The command to run in the container. - * - * @example ['npm', 'install'] - * - * @see https://docs.docker.com/engine/reference/run/ - * - * @default - run the command defined in the image - */ - readonly command?: string[]; - - /** - * Additional Docker volumes to mount. - * - * @default - no additional volumes are mounted - */ - readonly volumes?: DockerVolume[]; - - /** - * The environment variables to pass to the container. - * - * @default - no environment variables. - */ - readonly environment?: { [key: string]: string; }; - - /** - * Working directory inside the container. - * - * @default /asset-input - */ - readonly workingDirectory?: string; /** * How to calculate the source hash for this asset. * * @default BUNDLE */ - readonly hashCalculation?: AssetHashCalculation; + readonly assetHashType?: AssetHashType; } /** * Source hash calculation */ -export enum AssetHashCalculation { +export enum AssetHashType { /** * Based on the content of the source path */ @@ -92,6 +48,11 @@ export enum AssetHashCalculation { * Based on the content of the bundled path */ BUNDLE = 'bundle', + + /** + * Use a custom hash + */ + CUSTOM = 'custom', } /** @@ -170,17 +131,17 @@ export class AssetStaging extends Construct { workingDirectory: props.bundling.workingDirectory ?? BUNDLING_INPUT_DIR, }); } catch (err) { - throw new Error(`Failed to run bundling Docker image for asset ${this.node.id}: ${err}`); + throw new Error(`Failed to run bundling Docker image for asset ${this.node.path}: ${err}`); } if (FileSystem.isEmpty(this.bundleDir)) { throw new Error(`Bundling did not produce any output. Check that your container writes content to ${BUNDLING_OUTPUT_DIR}.`); } - const hashCalculation = props.bundling.hashCalculation ?? AssetHashCalculation.BUNDLE; - if (hashCalculation === AssetHashCalculation.SOURCE) { + const hashCalculation = props.assetHashType ?? AssetHashType.BUNDLE; + if (hashCalculation === AssetHashType.SOURCE) { this.sourceHash = this.fingerprint(this.sourcePath); - } else if (hashCalculation === AssetHashCalculation.BUNDLE) { + } else if (hashCalculation === AssetHashType.BUNDLE) { this.sourceHash = this.fingerprint(this.bundleDir); } else { throw new Error('Unknown source hash calculation.'); @@ -188,10 +149,13 @@ export class AssetStaging extends Construct { if (stagingDisabled) { this.stagedPath = this.bundleDir; } else { - this.relativePath = `asset.${this.fingerprint(this.bundleDir)}`; // always bundle based + // Make relative path always bundle based. This way we move it + // in `synthesize()` to the staging directory only if the final + // bundled asset has changed and we can safely skip this otherwise. + this.relativePath = `asset.${this.fingerprint(this.bundleDir)}`; this.stagedPath = this.relativePath; // always relative to outdir } - } else { + } else { // No bundling this.sourceHash = this.fingerprint(this.sourcePath); if (stagingDisabled) { this.stagedPath = this.sourcePath; @@ -224,7 +188,8 @@ export class AssetStaging extends Construct { return; } catch (err) { // /tmp and cdk.out could be mounted across different mount points - // in this case we will fallback to copying. This can happen in WSL. + // in this case we will fallback to copying. This can happen in Windows + // Subsystem for Linux (WSL). if (err.code === 'EXDEV') { fs.mkdirSync(targetPath); FileSystem.copyDirectory(this.bundleDir, targetPath, this.fingerprintOptions); diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index 5da07fe481787..bfff68b40f5cd 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -4,35 +4,31 @@ export const BUNDLING_INPUT_DIR = '/asset-input'; export const BUNDLING_OUTPUT_DIR = '/asset-output'; /** - * A Docker volume + * Bundling options + * + * @experimental */ -export interface DockerVolume { +export interface BundlingOptions { /** - * The path to the file or directory on the host machine + * The Docker image where the command will run. */ - readonly hostPath: string; + readonly image: BundlingDockerImage; - /** - * The path where the file or directory is mounted in the container - */ - readonly containerPath: string; -} - -/** - * Docker run options - */ -interface DockerRunOptions { /** * The command to run in the container. * + * @example ['npm', 'install'] + * + * @see https://docs.docker.com/engine/reference/run/ + * * @default - run the command defined in the image */ readonly command?: string[]; /** - * Docker volumes to mount. + * Additional Docker volumes to mount. * - * @default - no volumes are mounted + * @default - no additional volumes are mounted */ readonly volumes?: DockerVolume[]; @@ -46,23 +42,11 @@ interface DockerRunOptions { /** * Working directory inside the container. * - * @default - image default + * @default /asset-input */ readonly workingDirectory?: string; } -/** - * Docker build options - */ -export interface DockerBuildOptions { - /** - * Build args - * - * @default - no build args - */ - readonly buildArgs?: { [key: string]: string }; -} - /** * A Docker image used for asset bundling */ @@ -130,6 +114,66 @@ export class BundlingDockerImage { } } +/** + * A Docker volume + */ +export interface DockerVolume { + /** + * The path to the file or directory on the host machine + */ + readonly hostPath: string; + + /** + * The path where the file or directory is mounted in the container + */ + readonly containerPath: string; +} + +/** + * Docker run options + */ +interface DockerRunOptions { + /** + * The command to run in the container. + * + * @default - run the command defined in the image + */ + readonly command?: string[]; + + /** + * Docker volumes to mount. + * + * @default - no volumes are mounted + */ + readonly volumes?: DockerVolume[]; + + /** + * The environment variables to pass to the container. + * + * @default - no environment variables. + */ + readonly environment?: { [key: string]: string; }; + + /** + * Working directory inside the container. + * + * @default - image default + */ + readonly workingDirectory?: string; +} + +/** + * Docker build options + */ +export interface DockerBuildOptions { + /** + * Build args + * + * @default - no build args + */ + readonly buildArgs?: { [key: string]: string }; +} + function flatten(x: string[][]) { return Array.prototype.concat([], ...x); } From 1c0d0b6246fa4fa7a1b43a4381597071da982466 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 8 Jun 2020 17:22:39 +0200 Subject: [PATCH 36/53] will be zipped --- packages/@aws-cdk/core/lib/asset-staging.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 99b22b651b241..2ac055199a842 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -19,7 +19,8 @@ export interface AssetStagingProps extends FingerprintOptions { * Bundle the asset by executing a command in a Docker container. * The asset path will be mounted at `/asset-input`. The Docker * container is responsible for putting content at `/asset-output`. - * The content at `/asset-output` will used as the final asset. + * The content at `/asset-output` will be zipped ans used as the + * final asset. * * @default - source is copied to staging directory * From 7399c1ce75ead686172d62b4004526d1f8d5faed Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 8 Jun 2020 17:34:53 +0200 Subject: [PATCH 37/53] always default to SOURCE --- packages/@aws-cdk/aws-lambda/test/integ.bundling.ts | 3 +-- packages/@aws-cdk/core/lib/asset-staging.ts | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts index c6add465805ba..6c1715bd05747 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts +++ b/packages/@aws-cdk/aws-lambda/test/integ.bundling.ts @@ -1,4 +1,4 @@ -import { App, AssetHashType, CfnOutput, Construct, Stack, StackProps } from '@aws-cdk/core'; +import { App, CfnOutput, Construct, Stack, StackProps } from '@aws-cdk/core'; import * as path from 'path'; import * as lambda from '../lib'; @@ -26,7 +26,6 @@ class TestStack extends Stack { ].join(' && '), ], }, - assetHashType: AssetHashType.SOURCE, }), runtime: lambda.Runtime.PYTHON_3_6, handler: 'index.handler', diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 2ac055199a842..da03b4a923665 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -139,8 +139,8 @@ export class AssetStaging extends Construct { throw new Error(`Bundling did not produce any output. Check that your container writes content to ${BUNDLING_OUTPUT_DIR}.`); } - const hashCalculation = props.assetHashType ?? AssetHashType.BUNDLE; - if (hashCalculation === AssetHashType.SOURCE) { + const hashCalculation = props.assetHashType ?? AssetHashType.SOURCE; + if (hashCalculation === AssetHashType.SOURCE || hashCalculation === AssetHashType.CUSTOM) { this.sourceHash = this.fingerprint(this.sourcePath); } else if (hashCalculation === AssetHashType.BUNDLE) { this.sourceHash = this.fingerprint(this.bundleDir); From b72aac63f342530787c1361a55204dc4145e81c1 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 8 Jun 2020 17:58:50 +0200 Subject: [PATCH 38/53] tests --- packages/@aws-cdk/core/test/test.staging.ts | 70 ++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/test/test.staging.ts b/packages/@aws-cdk/core/test/test.staging.ts index 3faeea3e95396..7b1c4ac13404d 100644 --- a/packages/@aws-cdk/core/test/test.staging.ts +++ b/packages/@aws-cdk/core/test/test.staging.ts @@ -2,7 +2,7 @@ import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs'; import { Test } from 'nodeunit'; import * as path from 'path'; -import { App, AssetStaging, Stack } from '../lib'; +import { App, AssetStaging, AssetHashType, BundlingDockerImage, Stack } from '../lib'; export = { 'base case'(test: Test) { @@ -74,4 +74,72 @@ export = { test.deepEqual(withExtra.sourceHash, 'c95c915a5722bb9019e2c725d11868e5a619b55f36172f76bcbcaa8bb2d10c5f'); test.done(); }, + + 'with bundling'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // WHEN + const asset = new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: BundlingDockerImage.fromRegistry('alpine'), + command: ['touch', '/asset-output/test.txt'], + }, + }); + + // THEN + const assembly = app.synth(); + test.deepEqual(fs.readdirSync(assembly.directory), [ + 'asset.33cbf2cae5432438e0f046bc45ba8c3cef7b6afcf47b59d1c183775c1918fb1f', // Bundle based + 'cdk.out', + 'manifest.json', + 'stack.template.json', + 'tree.json', + ]); + + test.equal(asset.sourceHash, '2f37f937c51e2c191af66acf9b09f548926008ec68c575bd2ee54b6e997c0e00'); // Source based + + test.done(); + }, + + 'bundling throws when /asset-ouput is empty'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // THEN + test.throws(() => new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: BundlingDockerImage.fromRegistry('alpine'), + }, + }), /Bundling did not produce any output/); + + test.done(); + }, + + 'bundling with BUNDLE asset hash type'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // WHEN + const asset = new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: BundlingDockerImage.fromRegistry('alpine'), + command: ['touch', '/asset-output/test.txt'], + }, + assetHashType: AssetHashType.BUNDLE, + }); + + test.equal(asset.sourceHash, '33cbf2cae5432438e0f046bc45ba8c3cef7b6afcf47b59d1c183775c1918fb1f') + + test.done(); + }, }; From 78022f6495b5cc2e53504bb6dbf36198446aac9f Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 8 Jun 2020 18:02:45 +0200 Subject: [PATCH 39/53] assetHash --- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 2 +- packages/@aws-cdk/core/lib/asset-staging.ts | 21 ++++++++++++++------ packages/@aws-cdk/core/test/test.staging.ts | 4 ++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index d36ae4646ac74..f91770b3def74 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -149,7 +149,7 @@ export class Asset extends cdk.Construct implements cdk.IAsset { assetHashType: props.assetHashType, }); - this.sourceHash = props.assetHash ?? props.sourceHash ?? staging.sourceHash; + this.sourceHash = props.assetHash ?? props.sourceHash ?? staging.assetHash; this.assetPath = staging.stagedPath; diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index da03b4a923665..f1074afd79dc7 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -19,7 +19,7 @@ export interface AssetStagingProps extends FingerprintOptions { * Bundle the asset by executing a command in a Docker container. * The asset path will be mounted at `/asset-input`. The Docker * container is responsible for putting content at `/asset-output`. - * The content at `/asset-output` will be zipped ans used as the + * The content at `/asset-output` will be zipped and used as the * final asset. * * @default - source is copied to staging directory @@ -89,10 +89,17 @@ export class AssetStaging extends Construct { public readonly sourcePath: string; /** - * A cryptographic hash of the source document(s). + * A cryptographic hash of the asset. + * + * @deprecated see `assetHash`. */ public readonly sourceHash: string; + /** + * A cryptographic hash of the asset. + */ + public readonly assetHash: string; + private readonly fingerprintOptions: FingerprintOptions; private readonly relativePath?: string; @@ -141,9 +148,9 @@ export class AssetStaging extends Construct { const hashCalculation = props.assetHashType ?? AssetHashType.SOURCE; if (hashCalculation === AssetHashType.SOURCE || hashCalculation === AssetHashType.CUSTOM) { - this.sourceHash = this.fingerprint(this.sourcePath); + this.assetHash = this.fingerprint(this.sourcePath); } else if (hashCalculation === AssetHashType.BUNDLE) { - this.sourceHash = this.fingerprint(this.bundleDir); + this.assetHash = this.fingerprint(this.bundleDir); } else { throw new Error('Unknown source hash calculation.'); } @@ -157,14 +164,16 @@ export class AssetStaging extends Construct { this.stagedPath = this.relativePath; // always relative to outdir } } else { // No bundling - this.sourceHash = this.fingerprint(this.sourcePath); + this.assetHash = this.fingerprint(this.sourcePath); if (stagingDisabled) { this.stagedPath = this.sourcePath; } else { - this.relativePath = `asset.${this.sourceHash}${path.extname(this.sourcePath)}`; + this.relativePath = `asset.${this.assetHash}${path.extname(this.sourcePath)}`; this.stagedPath = this.relativePath; // always relative to outdir } } + + this.sourceHash = this.assetHash; } protected synthesize(session: ISynthesisSession) { diff --git a/packages/@aws-cdk/core/test/test.staging.ts b/packages/@aws-cdk/core/test/test.staging.ts index 7b1c4ac13404d..5bd4d323e4d3a 100644 --- a/packages/@aws-cdk/core/test/test.staging.ts +++ b/packages/@aws-cdk/core/test/test.staging.ts @@ -100,7 +100,7 @@ export = { 'tree.json', ]); - test.equal(asset.sourceHash, '2f37f937c51e2c191af66acf9b09f548926008ec68c575bd2ee54b6e997c0e00'); // Source based + test.equal(asset.assetHash, '2f37f937c51e2c191af66acf9b09f548926008ec68c575bd2ee54b6e997c0e00'); // Source based test.done(); }, @@ -138,7 +138,7 @@ export = { assetHashType: AssetHashType.BUNDLE, }); - test.equal(asset.sourceHash, '33cbf2cae5432438e0f046bc45ba8c3cef7b6afcf47b59d1c183775c1918fb1f') + test.equal(asset.assetHash, '33cbf2cae5432438e0f046bc45ba8c3cef7b6afcf47b59d1c183775c1918fb1f') test.done(); }, From f646a00009d75a3e4e15870b4143ded75bcf65b4 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 8 Jun 2020 18:10:59 +0200 Subject: [PATCH 40/53] remove source --- packages/@aws-cdk/core/lib/asset-staging.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index f1074afd79dc7..e748d482e4e9e 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -29,7 +29,7 @@ export interface AssetStagingProps extends FingerprintOptions { readonly bundling?: BundlingOptions; /** - * How to calculate the source hash for this asset. + * How to calculate the hash for this asset. * * @default BUNDLE */ From 913144ab507d3af0a7367a600fd3548d6ed57c08 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 8 Jun 2020 18:11:18 +0200 Subject: [PATCH 41/53] tslint --- packages/@aws-cdk/core/test/test.staging.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/core/test/test.staging.ts b/packages/@aws-cdk/core/test/test.staging.ts index 5bd4d323e4d3a..41f8a52876cbd 100644 --- a/packages/@aws-cdk/core/test/test.staging.ts +++ b/packages/@aws-cdk/core/test/test.staging.ts @@ -2,7 +2,7 @@ import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs'; import { Test } from 'nodeunit'; import * as path from 'path'; -import { App, AssetStaging, AssetHashType, BundlingDockerImage, Stack } from '../lib'; +import { App, AssetHashType, AssetStaging, BundlingDockerImage, Stack } from '../lib'; export = { 'base case'(test: Test) { @@ -138,7 +138,7 @@ export = { assetHashType: AssetHashType.BUNDLE, }); - test.equal(asset.assetHash, '33cbf2cae5432438e0f046bc45ba8c3cef7b6afcf47b59d1c183775c1918fb1f') + test.equal(asset.assetHash, '33cbf2cae5432438e0f046bc45ba8c3cef7b6afcf47b59d1c183775c1918fb1f'); test.done(); }, From 1cdf0874203f26dbc2d5fb9d8cc30755c67d66c2 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 8 Jun 2020 18:15:33 +0200 Subject: [PATCH 42/53] correct doc default --- packages/@aws-cdk/core/lib/asset-staging.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index e748d482e4e9e..2be3af6867a34 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -31,7 +31,7 @@ export interface AssetStagingProps extends FingerprintOptions { /** * How to calculate the hash for this asset. * - * @default BUNDLE + * @default AssetHashType.SOURCE */ readonly assetHashType?: AssetHashType; } From 1eb32bd9154633de88fff60c6e0520b743832c8e Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 8 Jun 2020 19:15:35 +0200 Subject: [PATCH 43/53] fix build --- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index f91770b3def74..5b892a9a0cc45 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -5,7 +5,6 @@ import * as cdk from '@aws-cdk/core'; import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs'; import * as path from 'path'; -import { AssetHashType } from '@aws-cdk/core'; const ARCHIVE_EXTENSIONS = [ '.zip', '.jar' ]; @@ -23,7 +22,7 @@ export interface AssetOptions extends assets.CopyOptions { * be set to `AssetHashType.CUSTOM`. * * @default - based on `assetHashType` - */ + */ readonly assetHash?: string; /** @@ -34,7 +33,7 @@ export interface AssetOptions extends assets.CopyOptions { * * @default - the default is `AssetHashType.SOURCE`, but if `assetHash` is * explicitly specified this value defaults to `AssetHashType.CUSTOM`. - */ + */ readonly assetHashType?: cdk.AssetHashType; /** @@ -136,7 +135,7 @@ export class Asset extends cdk.Construct implements cdk.IAsset { constructor(scope: cdk.Construct, id: string, props: AssetProps) { super(scope, id); - if ((props.assetHash || props.sourceHash) && props.assetHashType && props.assetHashType !== AssetHashType.CUSTOM) { + if ((props.assetHash || props.sourceHash) && props.assetHashType && props.assetHashType !== cdk.AssetHashType.CUSTOM) { throw new Error(`Cannot specify \`${props.assetHashType}\` for \`assetHashType\` when \`assetHash\` is specified. Use \`CUSTOM\` or leave \`undefined\`.`); } From 6d6bea4a4274e7208026380c1c8ae4709e1d2fd5 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 8 Jun 2020 21:29:27 +0200 Subject: [PATCH 44/53] update integ test --- .../test/integ.assets.bundling.lit.expected.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json index 9af3702475302..21d2d76dbd488 100644 --- a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json +++ b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json @@ -1,16 +1,16 @@ { "Parameters": { - "AssetParameters4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cbS3Bucket6A787065": { + "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8S3Bucket8CD0F73B": { "Type": "String", - "Description": "S3 bucket for asset \"4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cb\"" + "Description": "S3 bucket for asset \"10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8\"" }, - "AssetParameters4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cbS3VersionKey61D8476E": { + "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8S3VersionKeyA9EAF743": { "Type": "String", - "Description": "S3 key for asset version \"4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cb\"" + "Description": "S3 key for asset version \"10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8\"" }, - "AssetParameters4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cbArtifactHash83AE1BC1": { + "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8ArtifactHashBAE492DD": { "Type": "String", - "Description": "Artifact hash for asset \"4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cb\"" + "Description": "Artifact hash for asset \"10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8\"" } }, "Resources": { @@ -40,7 +40,7 @@ }, ":s3:::", { - "Ref": "AssetParameters4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cbS3Bucket6A787065" + "Ref": "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8S3Bucket8CD0F73B" } ] ] @@ -55,7 +55,7 @@ }, ":s3:::", { - "Ref": "AssetParameters4f0a73620441d4ecb45db9aa49637c5696cbd960cd8f249c66e285c4cfd612cbS3Bucket6A787065" + "Ref": "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8S3Bucket8CD0F73B" }, "/*" ] From 39092a0962dbb73dc7d97a1c931dd47469b32851 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 8 Jun 2020 21:29:57 +0200 Subject: [PATCH 45/53] more tests to fix low coverage --- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 4 ++- .../@aws-cdk/aws-s3-assets/test/asset.test.ts | 31 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 5b892a9a0cc45..9c84d1a762b18 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -145,7 +145,9 @@ export class Asset extends cdk.Construct implements cdk.IAsset { exclude: props.exclude, follow: assets.toSymlinkFollow(props.follow), bundling: props.bundling, - assetHashType: props.assetHashType, + assetHashType: props.assetHashType ?? props.assetHash + ? cdk.AssetHashType.CUSTOM + : cdk.AssetHashType.SOURCE, }); this.sourceHash = props.assetHash ?? props.sourceHash ?? staging.assetHash; diff --git a/packages/@aws-cdk/aws-s3-assets/test/asset.test.ts b/packages/@aws-cdk/aws-s3-assets/test/asset.test.ts index 4da45143c59f8..837d5b220caac 100644 --- a/packages/@aws-cdk/aws-s3-assets/test/asset.test.ts +++ b/packages/@aws-cdk/aws-s3-assets/test/asset.test.ts @@ -323,6 +323,37 @@ describe('staging', () => { }); }); +test('throws when specifying assetHash with incorrect assetHashType', () => { + // GIVEN + const stack = new cdk.Stack(); + + // THEN + expect(() => new Asset(stack, 'MyAsset', { + path: SAMPLE_ASSET_DIR, + assetHash: 'my-custom-hash', + assetHashType: cdk.AssetHashType.SOURCE, + })).toThrow(/Cannot specify `source` for `assetHashType`/); +}); + +test('can use a custom hash', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + new Asset(stack, 'MyAsset', { + path: SAMPLE_ASSET_DIR, + assetHash: 'my-custom-hash', + }); + + // THEN + const entry = stack.node.metadata.find(m => m.type === 'aws:cdk:asset'); + expect(entry).toBeTruthy(); + + expect(stack.resolve(entry!.data)).toEqual(expect.objectContaining({ + sourceHash: 'my-custom-hash', + })); +}); + function mkdtempSync() { return fs.mkdtempSync(path.join(os.tmpdir(), 'assets.test')); } From f9b486c918b73bf77c72c00f7f06edb16d82d164 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Tue, 9 Jun 2020 09:39:46 +0200 Subject: [PATCH 46/53] make it clean! --- packages/@aws-cdk/assets/lib/index.ts | 1 - packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 53 +++--- .../@aws-cdk/aws-s3-assets/test/asset.test.ts | 31 ---- packages/@aws-cdk/core/lib/asset-staging.ts | 163 +++++++----------- packages/@aws-cdk/core/lib/assets.ts | 74 +++++++- packages/@aws-cdk/core/lib/fs/index.ts | 2 +- packages/@aws-cdk/core/test/test.staging.ts | 90 +++++++++- 7 files changed, 247 insertions(+), 167 deletions(-) diff --git a/packages/@aws-cdk/assets/lib/index.ts b/packages/@aws-cdk/assets/lib/index.ts index 0ad070b4d52e6..e2a67003867bd 100644 --- a/packages/@aws-cdk/assets/lib/index.ts +++ b/packages/@aws-cdk/assets/lib/index.ts @@ -2,4 +2,3 @@ export * from './api'; export * from './fs/follow-mode'; export * from './fs/options'; export * from './staging'; -export * from './compat'; diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 9c84d1a762b18..c49e843f0cc6f 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -8,7 +8,7 @@ import * as path from 'path'; const ARCHIVE_EXTENSIONS = [ '.zip', '.jar' ]; -export interface AssetOptions extends assets.CopyOptions { +export interface AssetOptions extends assets.CopyOptions, cdk.AssetOptions { /** * A list of principals that should be able to read this asset from S3. * You can use `asset.grantRead(principal)` to grant read permissions later. @@ -17,25 +17,6 @@ export interface AssetOptions extends assets.CopyOptions { */ readonly readers?: iam.IGrantable[]; - /** - * Specify a custom hash for this asset. If `assetHashType` is set it must - * be set to `AssetHashType.CUSTOM`. - * - * @default - based on `assetHashType` - */ - readonly assetHash?: string; - - /** - * Specifies the type of hash to calculate for this asset. - * - * If `assetHash` is configured, this option must be `undefined` or - * `AssetHashType.CUSTOM`. - * - * @default - the default is `AssetHashType.SOURCE`, but if `assetHash` is - * explicitly specified this value defaults to `AssetHashType.CUSTOM`. - */ - readonly assetHashType?: cdk.AssetHashType; - /** * Custom source hash to use when identifying the specific version of the asset. * @@ -130,27 +111,26 @@ export class Asset extends cdk.Construct implements cdk.IAsset { */ public readonly isZipArchive: boolean; + /** @deprecated see `assetHash` */ public readonly sourceHash: string; + public readonly assetHash: string; + constructor(scope: cdk.Construct, id: string, props: AssetProps) { super(scope, id); - if ((props.assetHash || props.sourceHash) && props.assetHashType && props.assetHashType !== cdk.AssetHashType.CUSTOM) { - throw new Error(`Cannot specify \`${props.assetHashType}\` for \`assetHashType\` when \`assetHash\` is specified. Use \`CUSTOM\` or leave \`undefined\`.`); - } - // stage the asset source (conditionally). const staging = new cdk.AssetStaging(this, 'Stage', { sourcePath: path.resolve(props.path), exclude: props.exclude, - follow: assets.toSymlinkFollow(props.follow), + follow: toSymlinkFollow(props.follow), + assetHash: props.assetHash ?? props.sourceHash, + assetHashType: props.assetHashType, bundling: props.bundling, - assetHashType: props.assetHashType ?? props.assetHash - ? cdk.AssetHashType.CUSTOM - : cdk.AssetHashType.SOURCE, }); - this.sourceHash = props.assetHash ?? props.sourceHash ?? staging.assetHash; + this.assetHash = staging.assetHash; + this.sourceHash = this.assetHash; this.assetPath = staging.stagedPath; @@ -236,3 +216,18 @@ function determinePackaging(assetPath: string): cdk.FileAssetPackaging { throw new Error(`Asset ${assetPath} is expected to be either a directory or a regular file`); } + +function toSymlinkFollow(follow?: assets.FollowMode): cdk.SymlinkFollowMode | undefined { + if (!follow) { + return undefined; + } + + switch (follow) { + case assets.FollowMode.NEVER: return cdk.SymlinkFollowMode.NEVER; + case assets.FollowMode.ALWAYS: return cdk.SymlinkFollowMode.ALWAYS; + case assets.FollowMode.BLOCK_EXTERNAL: return cdk.SymlinkFollowMode.BLOCK_EXTERNAL; + case assets.FollowMode.EXTERNAL: return cdk.SymlinkFollowMode.EXTERNAL; + default: + throw new Error(`unknown follow mode: ${follow}`); + } +} diff --git a/packages/@aws-cdk/aws-s3-assets/test/asset.test.ts b/packages/@aws-cdk/aws-s3-assets/test/asset.test.ts index 837d5b220caac..4da45143c59f8 100644 --- a/packages/@aws-cdk/aws-s3-assets/test/asset.test.ts +++ b/packages/@aws-cdk/aws-s3-assets/test/asset.test.ts @@ -323,37 +323,6 @@ describe('staging', () => { }); }); -test('throws when specifying assetHash with incorrect assetHashType', () => { - // GIVEN - const stack = new cdk.Stack(); - - // THEN - expect(() => new Asset(stack, 'MyAsset', { - path: SAMPLE_ASSET_DIR, - assetHash: 'my-custom-hash', - assetHashType: cdk.AssetHashType.SOURCE, - })).toThrow(/Cannot specify `source` for `assetHashType`/); -}); - -test('can use a custom hash', () => { - // GIVEN - const stack = new cdk.Stack(); - - // WHEN - new Asset(stack, 'MyAsset', { - path: SAMPLE_ASSET_DIR, - assetHash: 'my-custom-hash', - }); - - // THEN - const entry = stack.node.metadata.find(m => m.type === 'aws:cdk:asset'); - expect(entry).toBeTruthy(); - - expect(stack.resolve(entry!.data)).toEqual(expect.objectContaining({ - sourceHash: 'my-custom-hash', - })); -}); - function mkdtempSync() { return fs.mkdtempSync(path.join(os.tmpdir(), 'assets.test')); } diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 2be3af6867a34..f89e16f510c4c 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -2,6 +2,7 @@ import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; +import { AssetHashType, AssetOptions } from './assets'; import { BUNDLING_INPUT_DIR, BUNDLING_OUTPUT_DIR, BundlingOptions } from './bundling'; import { Construct, ISynthesisSession } from './construct-compat'; import { FileSystem, FingerprintOptions } from './fs'; @@ -9,51 +10,11 @@ import { FileSystem, FingerprintOptions } from './fs'; /** * Initialization properties for `AssetStaging`. */ -export interface AssetStagingProps extends FingerprintOptions { +export interface AssetStagingProps extends FingerprintOptions, AssetOptions { /** * The source file or directory to copy from. */ readonly sourcePath: string; - - /** - * Bundle the asset by executing a command in a Docker container. - * The asset path will be mounted at `/asset-input`. The Docker - * container is responsible for putting content at `/asset-output`. - * The content at `/asset-output` will be zipped and used as the - * final asset. - * - * @default - source is copied to staging directory - * - * @experimental - */ - readonly bundling?: BundlingOptions; - - /** - * How to calculate the hash for this asset. - * - * @default AssetHashType.SOURCE - */ - readonly assetHashType?: AssetHashType; -} - -/** - * Source hash calculation - */ -export enum AssetHashType { - /** - * Based on the content of the source path - */ - SOURCE = 'source', - - /** - * Based on the content of the bundled path - */ - BUNDLE = 'bundle', - - /** - * Use a custom hash - */ - CUSTOM = 'custom', } /** @@ -109,68 +70,42 @@ export class AssetStaging extends Construct { constructor(scope: Construct, id: string, props: AssetStagingProps) { super(scope, id); + if (props.assetHash && props.assetHashType && props.assetHashType !== AssetHashType.CUSTOM) { + throw new Error(`Cannot specify \`${props.assetHashType}\` for \`assetHashType\` when \`assetHash\` is specified. Use \`CUSTOM\` or leave \`undefined\`.`); + } + this.sourcePath = props.sourcePath; this.fingerprintOptions = props; - const stagingDisabled = this.node.tryGetContext(cxapi.DISABLE_ASSET_STAGING_CONTEXT); - if (props.bundling) { - // Create temporary directory for bundling - this.bundleDir = fs.mkdtempSync(path.resolve(path.join(os.tmpdir(), 'cdk-asset-bundle-'))); - - // Always mount input and output dir - const volumes = [ - { - hostPath: this.sourcePath, - containerPath: BUNDLING_INPUT_DIR, - }, - { - hostPath: this.bundleDir, - containerPath: BUNDLING_OUTPUT_DIR, - }, - ...props.bundling.volumes ?? [], - ]; + this.bundleDir = this.bundle(props.bundling); + } - try { - props.bundling.image._run({ - command: props.bundling.command, - volumes, - environment: props.bundling.environment, - workingDirectory: props.bundling.workingDirectory ?? BUNDLING_INPUT_DIR, - }); - } catch (err) { - throw new Error(`Failed to run bundling Docker image for asset ${this.node.path}: ${err}`); + const hashCalculation = props.assetHash ? AssetHashType.CUSTOM : props.assetHashType ?? AssetHashType.SOURCE; + if (hashCalculation === AssetHashType.SOURCE) { + this.assetHash = this.fingerprint(props.sourcePath); + } else if (hashCalculation === AssetHashType.BUNDLE) { + if (!this.bundleDir) { + throw new Error('Cannot use `AssetHashType.BUNDLE` when `bundling` is not specified.'); } - - if (FileSystem.isEmpty(this.bundleDir)) { - throw new Error(`Bundling did not produce any output. Check that your container writes content to ${BUNDLING_OUTPUT_DIR}.`); + this.assetHash = this.fingerprint(this.bundleDir); + } else if (hashCalculation === AssetHashType.CUSTOM) { + if (!props.assetHash) { + throw new Error('`assetHash` must be specified when `assetHashType` is set to `AssetHashType.CUSTOM`.'); } + this.assetHash = props.assetHash; + } else { + throw new Error('Unknown asset hash type.'); + } - const hashCalculation = props.assetHashType ?? AssetHashType.SOURCE; - if (hashCalculation === AssetHashType.SOURCE || hashCalculation === AssetHashType.CUSTOM) { - this.assetHash = this.fingerprint(this.sourcePath); - } else if (hashCalculation === AssetHashType.BUNDLE) { - this.assetHash = this.fingerprint(this.bundleDir); - } else { - throw new Error('Unknown source hash calculation.'); - } - if (stagingDisabled) { - this.stagedPath = this.bundleDir; - } else { - // Make relative path always bundle based. This way we move it - // in `synthesize()` to the staging directory only if the final - // bundled asset has changed and we can safely skip this otherwise. - this.relativePath = `asset.${this.fingerprint(this.bundleDir)}`; - this.stagedPath = this.relativePath; // always relative to outdir - } - } else { // No bundling - this.assetHash = this.fingerprint(this.sourcePath); - if (stagingDisabled) { - this.stagedPath = this.sourcePath; - } else { - this.relativePath = `asset.${this.assetHash}${path.extname(this.sourcePath)}`; - this.stagedPath = this.relativePath; // always relative to outdir - } + const stagingDisabled = this.node.tryGetContext(cxapi.DISABLE_ASSET_STAGING_CONTEXT); + if (stagingDisabled && this.bundleDir) { + this.stagedPath = this.bundleDir; + } else if (stagingDisabled) { + this.stagedPath = this.sourcePath; + } else { + this.relativePath = `asset.${this.assetHash}${path.extname(this.bundleDir ?? this.sourcePath)}`; + this.stagedPath = this.relativePath; } this.sourceHash = this.assetHash; @@ -184,8 +119,7 @@ export class AssetStaging extends Construct { const targetPath = path.join(session.assembly.outdir, this.relativePath); - // Already staged. This also works with bundling because relative - // path is always bundle based when bundling + // Already staged if (fs.existsSync(targetPath)) { return; } @@ -225,4 +159,39 @@ export class AssetStaging extends Construct { private fingerprint(fileOrDirectory: string) { return FileSystem.fingerprint(fileOrDirectory, this.fingerprintOptions); } + + private bundle(options: BundlingOptions): string { + // Create temporary directory for bundling + const bundleDir = fs.mkdtempSync(path.resolve(path.join(os.tmpdir(), 'cdk-asset-bundle-'))); + + // Always mount input and output dir + const volumes = [ + { + hostPath: this.sourcePath, + containerPath: BUNDLING_INPUT_DIR, + }, + { + hostPath: bundleDir, + containerPath: BUNDLING_OUTPUT_DIR, + }, + ...options.volumes ?? [], + ]; + + try { + options.image._run({ + command: options.command, + volumes, + environment: options.environment, + workingDirectory: options.workingDirectory ?? BUNDLING_INPUT_DIR, + }); + } catch (err) { + throw new Error(`Failed to run bundling Docker image for asset ${this.node.path}: ${err}`); + } + + if (FileSystem.isEmpty(bundleDir)) { + throw new Error(`Bundling did not produce any output. Check that your container writes content to ${BUNDLING_OUTPUT_DIR}.`); + } + + return bundleDir; + } } diff --git a/packages/@aws-cdk/core/lib/assets.ts b/packages/@aws-cdk/core/lib/assets.ts index 141251062a786..bad303dbd8c31 100644 --- a/packages/@aws-cdk/core/lib/assets.ts +++ b/packages/@aws-cdk/core/lib/assets.ts @@ -1,13 +1,79 @@ +import { BundlingOptions } from './bundling'; + /** * Common interface for all assets. */ export interface IAsset { /** - * A hash of the source of this asset, which is available at construction time. As this is a plain - * string, it can be used in construct IDs in order to enforce creation of a new resource when - * the content hash has changed. + * A hash of this asset, which is available at construction time. As this is a plain string, it + * can be used in construct IDs in order to enforce creation of a new resource when the content + * hash has changed. */ - readonly sourceHash: string; + readonly assetHash: string; +} + +/** + * Asset hash options + */ +export interface AssetOptions { + /** + * Specify a custom hash for this asset. If `assetHashType` is set it must + * be set to `AssetHashType.CUSTOM`. + * + * NOTE: the hash is used in order to identify a specific revision of the asset, and + * used for optimizing and caching deployment activities related to this asset such as + * packaging, uploading to Amazon S3, etc. If you chose to customize the hash, you will + * need to make sure it is updated every time the asset changes, or otherwise it is + * possible that some deployments will not be invalidated. + * + * @default - based on `assetHashType` + */ + readonly assetHash?: string; + + /** + * Specifies the type of hash to calculate for this asset. + * + * If `assetHash` is configured, this option must be `undefined` or + * `AssetHashType.CUSTOM`. + * + * @default - the default is `AssetHashType.SOURCE`, but if `assetHash` is + * explicitly specified this value defaults to `AssetHashType.CUSTOM`. + */ + readonly assetHashType?: AssetHashType; + + /** + * Bundle the asset by executing a command in a Docker container. + * The asset path will be mounted at `/asset-input`. The Docker + * container is responsible for putting content at `/asset-output`. + * The content at `/asset-output` will be zipped and used as the + * final asset. + * + * @default - uploaded as-is to S3 if the asset is a regular file or a .zip file, + * archived into a .zip file and uploaded to S3 otherwise + * + * @experimental + */ + readonly bundling?: BundlingOptions; +} + +/** + * The type of asset hash + */ +export enum AssetHashType { + /** + * Based on the content of the source path + */ + SOURCE = 'source', + + /** + * Based on the content of the bundled path + */ + BUNDLE = 'bundle', + + /** + * Use a custom hash + */ + CUSTOM = 'custom', } /** diff --git a/packages/@aws-cdk/core/lib/fs/index.ts b/packages/@aws-cdk/core/lib/fs/index.ts index 68f04a6727b08..01c6d132956e2 100644 --- a/packages/@aws-cdk/core/lib/fs/index.ts +++ b/packages/@aws-cdk/core/lib/fs/index.ts @@ -38,7 +38,7 @@ export class FileSystem { /** * Checks whether a directory is empty * - * @param dir The direcotry to check + * @param dir The directory to check */ public static isEmpty(dir: string): boolean { return fs.readdirSync(dir).length === 0; diff --git a/packages/@aws-cdk/core/test/test.staging.ts b/packages/@aws-cdk/core/test/test.staging.ts index 41f8a52876cbd..5d5ab521eba59 100644 --- a/packages/@aws-cdk/core/test/test.staging.ts +++ b/packages/@aws-cdk/core/test/test.staging.ts @@ -82,7 +82,7 @@ export = { const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); // WHEN - const asset = new AssetStaging(stack, 'Asset', { + new AssetStaging(stack, 'Asset', { sourcePath: directory, bundling: { image: BundlingDockerImage.fromRegistry('alpine'), @@ -93,15 +93,13 @@ export = { // THEN const assembly = app.synth(); test.deepEqual(fs.readdirSync(assembly.directory), [ - 'asset.33cbf2cae5432438e0f046bc45ba8c3cef7b6afcf47b59d1c183775c1918fb1f', // Bundle based + 'asset.2f37f937c51e2c191af66acf9b09f548926008ec68c575bd2ee54b6e997c0e00', 'cdk.out', 'manifest.json', 'stack.template.json', 'tree.json', ]); - test.equal(asset.assetHash, '2f37f937c51e2c191af66acf9b09f548926008ec68c575bd2ee54b6e997c0e00'); // Source based - test.done(); }, @@ -142,4 +140,88 @@ export = { test.done(); }, + + 'custom hash'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // WHEN + const asset = new AssetStaging(stack, 'Asset', { + sourcePath: directory, + assetHash: 'my-custom-hash', + }); + + test.equal(asset.assetHash, 'my-custom-hash'); + + test.done(); + }, + + 'throws with assetHash and not CUSTOM hash type'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // THEN + test.throws(() => new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: BundlingDockerImage.fromRegistry('alpine'), + command: ['touch', '/asset-output/test.txt'], + }, + assetHash: 'my-custom-hash', + assetHashType: AssetHashType.BUNDLE, + }), /Cannot specify `bundle` for `assetHashType`/); + + test.done(); + }, + + 'throws with BUNDLE hash type and no bundling'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // THEN + test.throws(() => new AssetStaging(stack, 'Asset', { + sourcePath: directory, + assetHashType: AssetHashType.BUNDLE, + }), /Cannot use `AssetHashType.BUNDLE` when `bundling` is not specified/); + + test.done(); + }, + + 'throws with CUSTOM and no hash'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // THEN + test.throws(() => new AssetStaging(stack, 'Asset', { + sourcePath: directory, + assetHashType: AssetHashType.CUSTOM, + }), /`assetHash` must be specified when `assetHashType` is set to `AssetHashType.CUSTOM`/); + + test.done(); + }, + + 'throws when bundling fails'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // THEN + test.throws(() => new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: BundlingDockerImage.fromRegistry('this-is-an-invalid-docker-image'), + }, + }), /Failed to run bundling Docker image for asset stack\/Asset/); + + test.done(); + }, }; From 61a7c19cf4777eeebb59bef1f8f0fa14edc64d20 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Tue, 9 Jun 2020 09:50:14 +0200 Subject: [PATCH 47/53] clean s3-asset --- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 25 +++++--------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index c49e843f0cc6f..8b9b2e4a0ea8b 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -32,20 +32,6 @@ export interface AssetOptions extends assets.CopyOptions, cdk.AssetOptions { * @deprecated see `assetHash` and `assetHashType` */ readonly sourceHash?: string; - - /** - * Bundle the asset by executing a command in a Docker container. - * The asset path will be mounted at `/asset-input`. The Docker - * container is responsible for putting content at `/asset-output`. - * The content at `/asset-output` will be zipped and used as the - * final asset. - * - * @default - uploaded as-is to S3 if the asset is a regular file or a .zip file, - * archived into a .zip file and uploaded to S3 otherwise - * - * @experimental - */ - readonly bundling?: cdk.BundlingOptions; } export interface AssetProps extends AssetOptions { @@ -111,7 +97,11 @@ export class Asset extends cdk.Construct implements cdk.IAsset { */ public readonly isZipArchive: boolean; - /** @deprecated see `assetHash` */ + /** + * A cryptographic hash of the asset. + * + * @deprecated see `assetHash` + */ public readonly sourceHash: string; public readonly assetHash: string; @@ -121,12 +111,9 @@ export class Asset extends cdk.Construct implements cdk.IAsset { // stage the asset source (conditionally). const staging = new cdk.AssetStaging(this, 'Stage', { + ...props, sourcePath: path.resolve(props.path), - exclude: props.exclude, follow: toSymlinkFollow(props.follow), - assetHash: props.assetHash ?? props.sourceHash, - assetHashType: props.assetHashType, - bundling: props.bundling, }); this.assetHash = staging.assetHash; From de3eb47456ebe22466950dc22e525fb3eb93f5a8 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Tue, 9 Jun 2020 09:52:13 +0200 Subject: [PATCH 48/53] sourceHash --- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 8b9b2e4a0ea8b..5e6ecb43f9dca 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -114,6 +114,7 @@ export class Asset extends cdk.Construct implements cdk.IAsset { ...props, sourcePath: path.resolve(props.path), follow: toSymlinkFollow(props.follow), + assetHash: props.assetHash ?? props.sourceHash, }); this.assetHash = staging.assetHash; From 2795e8775e380f55c41d3c681ebe110c0fc9a937 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Tue, 9 Jun 2020 10:00:10 +0200 Subject: [PATCH 49/53] fix coverage in s3-assets --- packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 16 +--------------- packages/@aws-cdk/aws-s3-assets/lib/compat.ts | 17 +++++++++++++++++ .../@aws-cdk/aws-s3-assets/test/compat.test.ts | 11 +++++++++++ 3 files changed, 29 insertions(+), 15 deletions(-) create mode 100644 packages/@aws-cdk/aws-s3-assets/lib/compat.ts create mode 100644 packages/@aws-cdk/aws-s3-assets/test/compat.test.ts diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 5e6ecb43f9dca..5c3f0a514f07e 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -5,6 +5,7 @@ import * as cdk from '@aws-cdk/core'; import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs'; import * as path from 'path'; +import { toSymlinkFollow } from './compat'; const ARCHIVE_EXTENSIONS = [ '.zip', '.jar' ]; @@ -204,18 +205,3 @@ function determinePackaging(assetPath: string): cdk.FileAssetPackaging { throw new Error(`Asset ${assetPath} is expected to be either a directory or a regular file`); } - -function toSymlinkFollow(follow?: assets.FollowMode): cdk.SymlinkFollowMode | undefined { - if (!follow) { - return undefined; - } - - switch (follow) { - case assets.FollowMode.NEVER: return cdk.SymlinkFollowMode.NEVER; - case assets.FollowMode.ALWAYS: return cdk.SymlinkFollowMode.ALWAYS; - case assets.FollowMode.BLOCK_EXTERNAL: return cdk.SymlinkFollowMode.BLOCK_EXTERNAL; - case assets.FollowMode.EXTERNAL: return cdk.SymlinkFollowMode.EXTERNAL; - default: - throw new Error(`unknown follow mode: ${follow}`); - } -} diff --git a/packages/@aws-cdk/aws-s3-assets/lib/compat.ts b/packages/@aws-cdk/aws-s3-assets/lib/compat.ts new file mode 100644 index 0000000000000..af080a15615a2 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-assets/lib/compat.ts @@ -0,0 +1,17 @@ +import { FollowMode } from '@aws-cdk/assets'; +import { SymlinkFollowMode } from '@aws-cdk/core'; + +export function toSymlinkFollow(follow?: FollowMode): SymlinkFollowMode | undefined { + if (!follow) { + return undefined; + } + + switch (follow) { + case FollowMode.NEVER: return SymlinkFollowMode.NEVER; + case FollowMode.ALWAYS: return SymlinkFollowMode.ALWAYS; + case FollowMode.BLOCK_EXTERNAL: return SymlinkFollowMode.BLOCK_EXTERNAL; + case FollowMode.EXTERNAL: return SymlinkFollowMode.EXTERNAL; + default: + throw new Error(`unknown follow mode: ${follow}`); + } +} diff --git a/packages/@aws-cdk/aws-s3-assets/test/compat.test.ts b/packages/@aws-cdk/aws-s3-assets/test/compat.test.ts new file mode 100644 index 0000000000000..41fbf0b57ac53 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-assets/test/compat.test.ts @@ -0,0 +1,11 @@ +import { FollowMode } from '@aws-cdk/assets'; +import { SymlinkFollowMode } from '@aws-cdk/core'; +import { toSymlinkFollow } from '../lib/compat'; + +test('FollowMode compatibility', () => { + expect(toSymlinkFollow(undefined)).toBeUndefined(); + expect(toSymlinkFollow(FollowMode.ALWAYS)).toBe(SymlinkFollowMode.ALWAYS); + expect(toSymlinkFollow(FollowMode.BLOCK_EXTERNAL)).toBe(SymlinkFollowMode.BLOCK_EXTERNAL); + expect(toSymlinkFollow(FollowMode.EXTERNAL)).toBe(SymlinkFollowMode.EXTERNAL); + expect(toSymlinkFollow(FollowMode.NEVER)).toBe(SymlinkFollowMode.NEVER); +}); From 3b77e7fb33367b53d16a3fc9e4525f8e2024f8d3 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Tue, 9 Jun 2020 10:17:53 +0200 Subject: [PATCH 50/53] extract to method + if/else --- packages/@aws-cdk/core/lib/asset-staging.ts | 46 ++++++++++++--------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index f89e16f510c4c..3642ac36a765f 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -81,28 +81,12 @@ export class AssetStaging extends Construct { this.bundleDir = this.bundle(props.bundling); } - const hashCalculation = props.assetHash ? AssetHashType.CUSTOM : props.assetHashType ?? AssetHashType.SOURCE; - if (hashCalculation === AssetHashType.SOURCE) { - this.assetHash = this.fingerprint(props.sourcePath); - } else if (hashCalculation === AssetHashType.BUNDLE) { - if (!this.bundleDir) { - throw new Error('Cannot use `AssetHashType.BUNDLE` when `bundling` is not specified.'); - } - this.assetHash = this.fingerprint(this.bundleDir); - } else if (hashCalculation === AssetHashType.CUSTOM) { - if (!props.assetHash) { - throw new Error('`assetHash` must be specified when `assetHashType` is set to `AssetHashType.CUSTOM`.'); - } - this.assetHash = props.assetHash; - } else { - throw new Error('Unknown asset hash type.'); - } + const hashType = props.assetHash ? AssetHashType.CUSTOM : props.assetHashType ?? AssetHashType.SOURCE; + this.assetHash = this.calculateHash(hashType, props.assetHash); const stagingDisabled = this.node.tryGetContext(cxapi.DISABLE_ASSET_STAGING_CONTEXT); - if (stagingDisabled && this.bundleDir) { - this.stagedPath = this.bundleDir; - } else if (stagingDisabled) { - this.stagedPath = this.sourcePath; + if (stagingDisabled) { + this.stagedPath = this.bundleDir ?? this.sourcePath; } else { this.relativePath = `asset.${this.assetHash}${path.extname(this.bundleDir ?? this.sourcePath)}`; this.stagedPath = this.relativePath; @@ -194,4 +178,26 @@ export class AssetStaging extends Construct { return bundleDir; } + + private calculateHash(hashType: AssetHashType, assetHash?: string): string { + if (hashType === AssetHashType.SOURCE) { + return this.fingerprint(this.sourcePath); + } + + if (hashType === AssetHashType.BUNDLE) { + if (!this.bundleDir) { + throw new Error('Cannot use `AssetHashType.BUNDLE` when `bundling` is not specified.'); + } + return this.fingerprint(this.bundleDir); + } + + if (hashType === AssetHashType.CUSTOM) { + if (!assetHash) { + throw new Error('`assetHash` must be specified when `assetHashType` is set to `AssetHashType.CUSTOM`.'); + } + return assetHash; + } + + throw new Error('Unknown asset hash type.'); + } } From 1ff601d37b2ecc3fc7e42a1fabb01a1f6054c642 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Tue, 9 Jun 2020 10:25:59 +0200 Subject: [PATCH 51/53] remove fingerprint() method --- packages/@aws-cdk/core/lib/asset-staging.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 3642ac36a765f..4719e5700dfac 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -140,10 +140,6 @@ export class AssetStaging extends Construct { } } - private fingerprint(fileOrDirectory: string) { - return FileSystem.fingerprint(fileOrDirectory, this.fingerprintOptions); - } - private bundle(options: BundlingOptions): string { // Create temporary directory for bundling const bundleDir = fs.mkdtempSync(path.resolve(path.join(os.tmpdir(), 'cdk-asset-bundle-'))); @@ -181,14 +177,14 @@ export class AssetStaging extends Construct { private calculateHash(hashType: AssetHashType, assetHash?: string): string { if (hashType === AssetHashType.SOURCE) { - return this.fingerprint(this.sourcePath); + return FileSystem.fingerprint(this.sourcePath, this.fingerprintOptions); } if (hashType === AssetHashType.BUNDLE) { if (!this.bundleDir) { throw new Error('Cannot use `AssetHashType.BUNDLE` when `bundling` is not specified.'); } - return this.fingerprint(this.bundleDir); + return FileSystem.fingerprint(this.bundleDir, this.fingerprintOptions); } if (hashType === AssetHashType.CUSTOM) { From 17df113e5ef87d57656b6ca06ec165bbf7ab69b8 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Tue, 9 Jun 2020 10:36:33 +0200 Subject: [PATCH 52/53] switch --- packages/@aws-cdk/core/lib/asset-staging.ts | 40 ++++++++++----------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 4719e5700dfac..b2b1f37628922 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -70,10 +70,6 @@ export class AssetStaging extends Construct { constructor(scope: Construct, id: string, props: AssetStagingProps) { super(scope, id); - if (props.assetHash && props.assetHashType && props.assetHashType !== AssetHashType.CUSTOM) { - throw new Error(`Cannot specify \`${props.assetHashType}\` for \`assetHashType\` when \`assetHash\` is specified. Use \`CUSTOM\` or leave \`undefined\`.`); - } - this.sourcePath = props.sourcePath; this.fingerprintOptions = props; @@ -81,6 +77,9 @@ export class AssetStaging extends Construct { this.bundleDir = this.bundle(props.bundling); } + if (props.assetHash && props.assetHashType && props.assetHashType !== AssetHashType.CUSTOM) { + throw new Error(`Cannot specify \`${props.assetHashType}\` for \`assetHashType\` when \`assetHash\` is specified. Use \`CUSTOM\` or leave \`undefined\`.`); + } const hashType = props.assetHash ? AssetHashType.CUSTOM : props.assetHashType ?? AssetHashType.SOURCE; this.assetHash = this.calculateHash(hashType, props.assetHash); @@ -176,24 +175,21 @@ export class AssetStaging extends Construct { } private calculateHash(hashType: AssetHashType, assetHash?: string): string { - if (hashType === AssetHashType.SOURCE) { - return FileSystem.fingerprint(this.sourcePath, this.fingerprintOptions); - } - - if (hashType === AssetHashType.BUNDLE) { - if (!this.bundleDir) { - throw new Error('Cannot use `AssetHashType.BUNDLE` when `bundling` is not specified.'); - } - return FileSystem.fingerprint(this.bundleDir, this.fingerprintOptions); - } - - if (hashType === AssetHashType.CUSTOM) { - if (!assetHash) { - throw new Error('`assetHash` must be specified when `assetHashType` is set to `AssetHashType.CUSTOM`.'); - } - return assetHash; + switch (hashType) { + case AssetHashType.SOURCE: + return FileSystem.fingerprint(this.sourcePath, this.fingerprintOptions); + case AssetHashType.BUNDLE: + if (!this.bundleDir) { + throw new Error('Cannot use `AssetHashType.BUNDLE` when `bundling` is not specified.'); + } + return FileSystem.fingerprint(this.bundleDir, this.fingerprintOptions); + case AssetHashType.CUSTOM: + if (!assetHash) { + throw new Error('`assetHash` must be specified when `assetHashType` is set to `AssetHashType.CUSTOM`.'); + } + return assetHash; + default: + throw new Error('Unknown asset hash type.'); } - - throw new Error('Unknown asset hash type.'); } } From 96df801199260750d10b41bd6bf61f8d3670e3b3 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Tue, 9 Jun 2020 11:15:57 +0200 Subject: [PATCH 53/53] calculateHash(props: AssetStagingProps) --- packages/@aws-cdk/core/lib/asset-staging.ts | 25 ++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index b2b1f37628922..c37d8d441d7c0 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -77,11 +77,7 @@ export class AssetStaging extends Construct { this.bundleDir = this.bundle(props.bundling); } - if (props.assetHash && props.assetHashType && props.assetHashType !== AssetHashType.CUSTOM) { - throw new Error(`Cannot specify \`${props.assetHashType}\` for \`assetHashType\` when \`assetHash\` is specified. Use \`CUSTOM\` or leave \`undefined\`.`); - } - const hashType = props.assetHash ? AssetHashType.CUSTOM : props.assetHashType ?? AssetHashType.SOURCE; - this.assetHash = this.calculateHash(hashType, props.assetHash); + this.assetHash = this.calculateHash(props); const stagingDisabled = this.node.tryGetContext(cxapi.DISABLE_ASSET_STAGING_CONTEXT); if (stagingDisabled) { @@ -174,7 +170,20 @@ export class AssetStaging extends Construct { return bundleDir; } - private calculateHash(hashType: AssetHashType, assetHash?: string): string { + private calculateHash(props: AssetStagingProps): string { + let hashType: AssetHashType; + + if (props.assetHash) { + if (props.assetHashType && props.assetHashType !== AssetHashType.CUSTOM) { + throw new Error(`Cannot specify \`${props.assetHashType}\` for \`assetHashType\` when \`assetHash\` is specified. Use \`CUSTOM\` or leave \`undefined\`.`); + } + hashType = AssetHashType.CUSTOM; + } else if (props.assetHashType) { + hashType = props.assetHashType; + } else { + hashType = AssetHashType.SOURCE; + } + switch (hashType) { case AssetHashType.SOURCE: return FileSystem.fingerprint(this.sourcePath, this.fingerprintOptions); @@ -184,10 +193,10 @@ export class AssetStaging extends Construct { } return FileSystem.fingerprint(this.bundleDir, this.fingerprintOptions); case AssetHashType.CUSTOM: - if (!assetHash) { + if (!props.assetHash) { throw new Error('`assetHash` must be specified when `assetHashType` is set to `AssetHashType.CUSTOM`.'); } - return assetHash; + return props.assetHash; default: throw new Error('Unknown asset hash type.'); }