Skip to content

Commit

Permalink
feat(lambda): configurable retries for log retention custom resource (#…
Browse files Browse the repository at this point in the history 
…8258)

This prevents throttling issues on stacks with a lot of Lambdas.

fixes #8257

Implemented configurable `maxRetries` and `base` properties as part of the LogRetentionRetryOptions. The AWS SDK also supports specifying a customBackoff function. I skipped that as it's hard to implement in the current setup (impossible to provide a callback function in the event JSON).

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
  • Loading branch information
@jaapvanblaaderen
jaapvanblaaderen authored Jun 12, 2020
1 parent a56a4d9 commit e17a49a
Show file tree
Hide file tree
Showing 7 changed files with 135 additions and 29 deletions.
3 changes: 3 additions & 0 deletions packages/@aws-cdk/aws-lambda/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,9 @@ The `logRetention` property can be used to set a different expiration period.
It is possible to obtain the function's log group as a `logs.ILogGroup` by calling the `logGroup` property of the
`Function` construct.

By default, CDK uses the AWS SDK retry options when creating a log group. The `logRetentionRetryOptions` property
allows you to customize the maximum number of retries and base backoff duration.

*Note* that, if either `logRetention` is set or `logGroup` property is called, a [CloudFormation custom
resource](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cfn-customresource.html) is added
to the stack that pre-creates the log group as part of the stack deployment, if it already doesn't exist, and sets the
Expand Down
11 changes: 10 additions & 1 deletion packages/@aws-cdk/aws-lambda/lib/function.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { calculateFunctionHash, trimFromStart } from './function-hash';
import { Version, VersionOptions } from './lambda-version';
import { CfnFunction } from './lambda.generated';
import { ILayerVersion } from './layers';
import { LogRetention } from './log-retention';
import { LogRetention, LogRetentionRetryOptions } from './log-retention';
import { Runtime } from './runtime';

/**
Expand Down Expand Up @@ -232,6 +232,14 @@ export interface FunctionOptions extends EventInvokeConfigOptions {
*/
readonly logRetentionRole?: iam.IRole;

/**
* When log retention is specified, a custom resource attempts to create the CloudWatch log group.
* These options control the retry policy when interacting with CloudWatch APIs.
*
* @default - Default AWS SDK retry options.
*/
readonly logRetentionRetryOptions?: LogRetentionRetryOptions;

/**
* Options for the `lambda.Version` resource automatically created by the
* `fn.currentVersion` method.
Expand Down Expand Up @@ -544,6 +552,7 @@ export class Function extends FunctionBase {
logGroupName: `/aws/lambda/${this.functionName}`,
retention: props.logRetention,
role: props.logRetentionRole,
logRetentionRetryOptions: props.logRetentionRetryOptions,
});
this._logGroup = logs.LogGroup.fromLogGroupArn(this, 'LogGroup', logretention.logGroupArn);
}
Expand Down
45 changes: 36 additions & 9 deletions packages/@aws-cdk/aws-lambda/lib/log-retention-provider/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,23 @@

// eslint-disable-next-line import/no-extraneous-dependencies
import * as AWS from 'aws-sdk';
// eslint-disable-next-line import/no-extraneous-dependencies
import { RetryDelayOptions } from 'aws-sdk/lib/config';

interface SdkRetryOptions {
maxRetries?: number;
retryOptions?: RetryDelayOptions;
}

/**
* Creates a log group and doesn't throw if it exists.
*
* @param logGroupName the name of the log group to create
* @param logGroupName the name of the log group to create.
* @param options CloudWatch API SDK options.
*/
async function createLogGroupSafe(logGroupName: string) {
async function createLogGroupSafe(logGroupName: string, options?: SdkRetryOptions) {
try { // Try to create the log group
const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28' });
const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28', ...options });
await cloudwatchlogs.createLogGroup({ logGroupName }).promise();
} catch (e) {
if (e.code !== 'ResourceAlreadyExistsException') {
Expand All @@ -23,10 +31,11 @@ async function createLogGroupSafe(logGroupName: string) {
* Puts or deletes a retention policy on a log group.
*
* @param logGroupName the name of the log group to create
* @param options CloudWatch API SDK options.
* @param retentionInDays the number of days to retain the log events in the specified log group.
*/
async function setRetentionPolicy(logGroupName: string, retentionInDays?: number) {
const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28' });
async function setRetentionPolicy(logGroupName: string, options?: SdkRetryOptions, retentionInDays?: number) {
const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28', ...options });
if (!retentionInDays) {
await cloudwatchlogs.deleteRetentionPolicy({ logGroupName }).promise();
} else {
Expand All @@ -41,10 +50,13 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent
// The target log group
const logGroupName = event.ResourceProperties.LogGroupName;

// Parse to AWS SDK retry options
const retryOptions = parseRetryOptions(event.ResourceProperties.SdkRetry);

if (event.RequestType === 'Create' || event.RequestType === 'Update') {
// Act on the target log group
await createLogGroupSafe(logGroupName);
await setRetentionPolicy(logGroupName, parseInt(event.ResourceProperties.RetentionInDays, 10));
await createLogGroupSafe(logGroupName, retryOptions);
await setRetentionPolicy(logGroupName, retryOptions, parseInt(event.ResourceProperties.RetentionInDays, 10));

if (event.RequestType === 'Create') {
// Set a retention policy of 1 day on the logs of this function. The log
Expand All @@ -56,8 +68,8 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent
// same time. This can sometime result in an OperationAbortedException. To
// avoid this and because this operation is not critical we catch all errors.
try {
await createLogGroupSafe(`/aws/lambda/${context.functionName}`);
await setRetentionPolicy(`/aws/lambda/${context.functionName}`, 1);
await createLogGroupSafe(`/aws/lambda/${context.functionName}`, retryOptions);
await setRetentionPolicy(`/aws/lambda/${context.functionName}`, retryOptions, 1);
} catch (e) {
console.log(e);
}
Expand Down Expand Up @@ -108,4 +120,19 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent
}
});
}

function parseRetryOptions(rawOptions: any): SdkRetryOptions {
const retryOptions: SdkRetryOptions = {};
if (rawOptions) {
if (rawOptions.maxRetries) {
retryOptions.maxRetries = parseInt(rawOptions.maxRetries, 10);
}
if (rawOptions.base) {
retryOptions.retryOptions = {
base: parseInt(rawOptions.base, 10),
};
}
}
return retryOptions;
}
}
30 changes: 30 additions & 0 deletions packages/@aws-cdk/aws-lambda/lib/log-retention.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,31 @@ export interface LogRetentionProps {
* @default - A new role is created
*/
readonly role?: iam.IRole;

/**
* Retry options for all AWS API calls.
*
* @default - AWS SDK default retry options
*/
readonly logRetentionRetryOptions?: LogRetentionRetryOptions;
}

/**
* Retry options for all AWS API calls.
*/
export interface LogRetentionRetryOptions {
/**
* The maximum amount of retries.
*
* @default 3 (AWS SDK default)
*/
readonly maxRetries?: number;
/**
* The base duration to use in the exponential backoff for operation retries.
*
* @default Duration.millis(100) (AWS SDK default)
*/
readonly base?: cdk.Duration;
}

/**
Expand Down Expand Up @@ -64,11 +89,16 @@ export class LogRetention extends cdk.Construct {

// Need to use a CfnResource here to prevent lerna dependency cycles
// @aws-cdk/aws-cloudformation -> @aws-cdk/aws-lambda -> @aws-cdk/aws-cloudformation
const retryOptions = props.logRetentionRetryOptions;
const resource = new cdk.CfnResource(this, 'Resource', {
type: 'Custom::LogRetention',
properties: {
ServiceToken: provider.functionArn,
LogGroupName: props.logGroupName,
SdkRetry: retryOptions ? {
maxRetries: retryOptions.maxRetries,
base: retryOptions.base?.toMilliseconds(),
} : undefined,
RetentionInDays: props.retention === logs.RetentionDays.INFINITE ? undefined : props.retention,
},
});
Expand Down
18 changes: 9 additions & 9 deletions packages/@aws-cdk/aws-lambda/test/integ.log-retention.expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@
"Properties": {
"Code": {
"S3Bucket": {
"Ref": "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3Bucket7046E6CE"
"Ref": "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3Bucket21D86049"
},
"S3Key": {
"Fn::Join": [
Expand All @@ -146,7 +146,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3VersionKey3194A583"
"Ref": "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3VersionKey1F67C4C1"
}
]
}
Expand All @@ -159,7 +159,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3VersionKey3194A583"
"Ref": "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3VersionKey1F67C4C1"
}
]
}
Expand Down Expand Up @@ -331,17 +331,17 @@
}
},
"Parameters": {
"AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3Bucket7046E6CE": {
"AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3Bucket21D86049": {
"Type": "String",
"Description": "S3 bucket for asset \"82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4e\""
"Description": "S3 bucket for asset \"3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3\""
},
"AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3VersionKey3194A583": {
"AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3VersionKey1F67C4C1": {
"Type": "String",
"Description": "S3 key for asset version \"82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4e\""
"Description": "S3 key for asset version \"3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3\""
},
"AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eArtifactHashB967D42A": {
"AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3ArtifactHash31AA1F7C": {
"Type": "String",
"Description": "Artifact hash for asset \"82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4e\""
"Description": "Artifact hash for asset \"3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3\""
}
}
}
37 changes: 37 additions & 0 deletions packages/@aws-cdk/aws-lambda/test/test.log-retention-provider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,4 +296,41 @@ export = {

test.done();
},

async 'custom log retention retry options'(test: Test) {
AWS.mock('CloudWatchLogs', 'createLogGroup', sinon.fake.resolves({}));
AWS.mock('CloudWatchLogs', 'putRetentionPolicy', sinon.fake.resolves({}));
AWS.mock('CloudWatchLogs', 'deleteRetentionPolicy', sinon.fake.resolves({}));

const event = {
...eventCommon,
RequestType: 'Create',
ResourceProperties: {
ServiceToken: 'token',
RetentionInDays: '30',
LogGroupName: 'group',
SdkRetry: {
maxRetries: '5',
base: '300',
},
},
};

const request = createRequest('SUCCESS');

await provider.handler(event as AWSLambda.CloudFormationCustomResourceCreateEvent, context);

sinon.assert.calledWith(AWSSDK.CloudWatchLogs as any, {
apiVersion: '2014-03-28',
maxRetries: 5,
retryOptions: {
base: 300,
},
});

test.equal(request.isDone(), true);

test.done();
},

};
20 changes: 10 additions & 10 deletions packages/@aws-cdk/aws-rds/test/integ.instance.lit.expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -967,7 +967,7 @@
"Properties": {
"Code": {
"S3Bucket": {
"Ref": "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3Bucket7046E6CE"
"Ref": "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3Bucket21D86049"
},
"S3Key": {
"Fn::Join": [
Expand All @@ -980,7 +980,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3VersionKey3194A583"
"Ref": "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3VersionKey1F67C4C1"
}
]
}
Expand All @@ -993,7 +993,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3VersionKey3194A583"
"Ref": "AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3VersionKey1F67C4C1"
}
]
}
Expand Down Expand Up @@ -1108,17 +1108,17 @@
}
},
"Parameters": {
"AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3Bucket7046E6CE": {
"AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3Bucket21D86049": {
"Type": "String",
"Description": "S3 bucket for asset \"82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4e\""
"Description": "S3 bucket for asset \"3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3\""
},
"AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eS3VersionKey3194A583": {
"AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3S3VersionKey1F67C4C1": {
"Type": "String",
"Description": "S3 key for asset version \"82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4e\""
"Description": "S3 key for asset version \"3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3\""
},
"AssetParameters82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4eArtifactHashB967D42A": {
"AssetParameters3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3ArtifactHash31AA1F7C": {
"Type": "String",
"Description": "Artifact hash for asset \"82c54bfa7c42ba410d6d18dad983ba51c93a5ea940818c5c20230f8b59c19d4e\""
"Description": "Artifact hash for asset \"3974ceb096f16a0d6c372c0c821ca2ab0333112497b2d3bc462ccaf2fc6037c3\""
}
}
}
}

0 comments on commit e17a49a

Please sign in to comment.