From 6fc6b3e4bfee02bc42b4fb787be2b766d801986f Mon Sep 17 00:00:00 2001 From: mickychetta Date: Wed, 3 Nov 2021 23:37:24 +0000 Subject: [PATCH] added logS3AccessLogs and LoggingBucket props --- .../README.md | 5 +- .../lib/index.ts | 23 +- .../integ.customLoggingBucket.expected.json | 501 ++++++++++++++++++ .../test/integ.customLoggingBucket.ts | 64 +++ .../test/integ.noLoggingBucket.expected.json | 411 ++++++++++++++ .../test/integ.noLoggingBucket.ts | 58 ++ .../test.kinesisfirehose-analytics-s3.test.ts | 92 +++- 7 files changed, 1147 insertions(+), 7 deletions(-) create mode 100644 source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.customLoggingBucket.expected.json create mode 100644 source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.customLoggingBucket.ts create mode 100644 source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.noLoggingBucket.expected.json create mode 100644 source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.noLoggingBucket.ts diff --git a/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/README.md b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/README.md index 0c3093961..2b2d6140f 100644 --- a/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/README.md +++ b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/README.md @@ -83,6 +83,8 @@ _Parameters_ |existingBucketObj?|[`s3.IBucket`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.IBucket.html)|Existing instance of S3 Bucket object. If this is provided, then also providing bucketProps is an error. | |bucketProps?|[`s3.BucketProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.BucketProps.html)|User provided props to override the default props for the S3 Bucket.| |logGroupProps?|[`logs.LogGroupProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-logs.LogGroupProps.html)|User provided props to override the default props for for the CloudWatchLogs LogGroup.| +|loggingBucketProps?|[`s3.BucketProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.BucketProps.html)|Optional user provided props to override the default props for the S3 Logging Bucket.| +|logS3AccessLogs?| boolean|Whether to turn on Access Logging for the S3 bucket. Creates an S3 bucket with associated storage costs for the logs. Enabling Access Logging is a best practice. default - true| ## Pattern Properties @@ -90,10 +92,11 @@ _Parameters_ |:-------------|:----------------|-----------------| |kinesisAnalytics|[`kinesisAnalytics.CfnApplication`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kinesisanalytics.CfnApplication.html)|Returns an instance of the Kinesis Analytics application created by the pattern.| |kinesisFirehose|[`kinesisFirehose.CfnDeliveryStream`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kinesisfirehose.CfnDeliveryStream.html)|Returns an instance of the Kinesis Firehose delivery stream created by the pattern.| -|kinesisFirehoseRole|[`iam.Role`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-iam.Role.html)|Returns an instance of the iam.Role created by the construct for Kinesis Data Firehose delivery stream| +|kinesisFirehoseRole|[`iam.Role`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-iam.Role.html)|Returns an instance of the iam.Role created by the construct for Kinesis Data Firehose delivery stream.| |kinesisFirehoseLogGroup|[`logs.LogGroup`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-logs.LogGroup.html)|Returns an instance of the LogGroup created by the construct for Kinesis Data Firehose delivery stream| |s3Bucket?|[`s3.Bucket`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.Bucket.html)|Returns an instance of the S3 bucket created by the pattern.| |s3LoggingBucket?|[`s3.Bucket`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.Bucket.html)|Returns an instance of s3.Bucket created by the construct as the logging bucket for the primary bucket.| +|s3BucketInterface|[`s3.IBucket`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-s3.IBucket.html)|Returns an instance of s3.IBucket created by the construct.| ## Default settings diff --git a/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/lib/index.ts b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/lib/index.ts index cc8963db6..826f4f25b 100644 --- a/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/lib/index.ts +++ b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/lib/index.ts @@ -56,6 +56,19 @@ export interface KinesisFirehoseToAnalyticsAndS3Props { * @default - Default props are used */ readonly logGroupProps?: logs.LogGroupProps + /** + * Optional user provided props to override the default props for the S3 Logging Bucket. + * + * @default - Default props are used + */ + readonly loggingBucketProps?: s3.BucketProps; + /** + * Whether to turn on Access Logs for the S3 bucket with the associated storage costs. + * Enabling Access Logging is a best practice. + * + * @default - true + */ + readonly logS3AccessLogs?: boolean; } /** @@ -68,6 +81,7 @@ export class KinesisFirehoseToAnalyticsAndS3 extends Construct { public readonly kinesisFirehoseLogGroup: logs.LogGroup; public readonly s3Bucket?: s3.Bucket; public readonly s3LoggingBucket?: s3.Bucket; + public readonly s3BucketInterface: s3.IBucket; /** * @summary Constructs a new instance of the KinesisFirehoseToAnalyticsAndS3 class. @@ -81,16 +95,14 @@ export class KinesisFirehoseToAnalyticsAndS3 extends Construct { super(scope, id); defaults.CheckProps(props); - if (props.existingBucketObj && props.bucketProps) { - throw new Error('Cannot specify both bucket properties and an existing bucket'); - } - // Setup the kinesisfirehose-s3 pattern const kinesisFirehoseToS3Props: KinesisFirehoseToS3Props = { kinesisFirehoseProps: props.kinesisFirehoseProps, existingBucketObj: props.existingBucketObj, bucketProps: props.bucketProps, - logGroupProps: props.logGroupProps + logGroupProps: props.logGroupProps, + loggingBucketProps: props.loggingBucketProps, + logS3AccessLogs: props.logS3AccessLogs }; // Add the kinesisfirehose-s3 pattern @@ -107,5 +119,6 @@ export class KinesisFirehoseToAnalyticsAndS3 extends Construct { this.kinesisFirehoseRole = kfs.kinesisFirehoseRole; this.s3Bucket = kfs.s3Bucket; this.s3LoggingBucket = kfs.s3LoggingBucket; + this.s3BucketInterface = kfs.s3BucketInterface; } } \ No newline at end of file diff --git a/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.customLoggingBucket.expected.json b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.customLoggingBucket.expected.json new file mode 100644 index 000000000..95b0c21f2 --- /dev/null +++ b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.customLoggingBucket.expected.json @@ -0,0 +1,501 @@ +{ + "Resources": { + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3LoggingBucketE14ECC0A": { + "Type": "AWS::S3::Bucket", + "Properties": { + "AccessControl": "LogDeliveryWrite", + "BucketEncryption": { + "ServerSideEncryptionConfiguration": [ + { + "ServerSideEncryptionByDefault": { + "SSEAlgorithm": "AES256" + } + } + ] + }, + "BucketName": "custom-logging-bucket", + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": true, + "BlockPublicPolicy": true, + "IgnorePublicAcls": true, + "RestrictPublicBuckets": true + }, + "VersioningConfiguration": { + "Status": "Enabled" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "cfn_nag": { + "rules_to_suppress": [ + { + "id": "W35", + "reason": "This S3 bucket is used as the access logging bucket for another bucket" + } + ] + } + } + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3LoggingBucketPolicy7DF00567": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3LoggingBucketE14ECC0A" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": "*", + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + } + }, + "Effect": "Deny", + "Principal": { + "AWS": "*" + }, + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3LoggingBucketE14ECC0A", + "Arn" + ] + }, + "/*" + ] + ] + }, + { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3LoggingBucketE14ECC0A", + "Arn" + ] + } + ], + "Sid": "HttpsOnly" + } + ], + "Version": "2012-10-17" + } + } + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56": { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketEncryption": { + "ServerSideEncryptionConfiguration": [ + { + "ServerSideEncryptionByDefault": { + "SSEAlgorithm": "AES256" + } + } + ] + }, + "LifecycleConfiguration": { + "Rules": [ + { + "NoncurrentVersionTransitions": [ + { + "StorageClass": "GLACIER", + "TransitionInDays": 90 + } + ], + "Status": "Enabled" + } + ] + }, + "LoggingConfiguration": { + "DestinationBucketName": { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3LoggingBucketE14ECC0A" + } + }, + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": true, + "BlockPublicPolicy": true, + "IgnorePublicAcls": true, + "RestrictPublicBuckets": true + }, + "VersioningConfiguration": { + "Status": "Enabled" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketPolicy55E9C081": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": "*", + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + } + }, + "Effect": "Deny", + "Principal": { + "AWS": "*" + }, + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56", + "Arn" + ] + }, + "/*" + ] + ] + }, + { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56", + "Arn" + ] + } + ], + "Sid": "HttpsOnly" + } + ], + "Version": "2012-10-17" + } + } + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupA3650D3F": { + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain", + "Metadata": { + "cfn_nag": { + "rules_to_suppress": [ + { + "id": "W86", + "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely" + }, + { + "id": "W84", + "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)" + } + ] + } + } + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupfirehoselogstream508B1DD2": { + "Type": "AWS::Logs::LogStream", + "Properties": { + "LogGroupName": { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupA3650D3F" + } + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehoseRoleD36244BC": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehosePolicy9CE17DCF": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": "logs:PutLogEvents", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:", + { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupA3650D3F" + }, + ":log-stream:", + { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupfirehoselogstream508B1DD2" + } + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehosePolicy9CE17DCF", + "Roles": [ + { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehoseRoleD36244BC" + } + ] + } + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehose9D683007": { + "Type": "AWS::KinesisFirehose::DeliveryStream", + "Properties": { + "DeliveryStreamEncryptionConfigurationInput": { + "KeyType": "AWS_OWNED_CMK" + }, + "ExtendedS3DestinationConfiguration": { + "BucketARN": { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56", + "Arn" + ] + }, + "BufferingHints": { + "IntervalInSeconds": 300, + "SizeInMBs": 5 + }, + "CloudWatchLoggingOptions": { + "Enabled": true, + "LogGroupName": { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupA3650D3F" + }, + "LogStreamName": { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupfirehoselogstream508B1DD2" + } + }, + "CompressionFormat": "GZIP", + "EncryptionConfiguration": { + "KMSEncryptionConfig": { + "AWSKMSKeyARN": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":kms:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":alias/aws/s3" + ] + ] + } + } + }, + "RoleARN": { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehoseRoleD36244BC", + "Arn" + ] + } + } + } + }, + "testkinesisfirehoseanalyticss3KinesisAnalyticsRoleB66D81BD": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "kinesisanalytics.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "testkinesisfirehoseanalyticss3KinesisAnalyticsPolicy9CAEA1BF": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "firehose:DescribeDeliveryStream", + "firehose:Get*" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehose9D683007", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "testkinesisfirehoseanalyticss3KinesisAnalyticsPolicy9CAEA1BF", + "Roles": [ + { + "Ref": "testkinesisfirehoseanalyticss3KinesisAnalyticsRoleB66D81BD" + } + ] + } + }, + "testkinesisfirehoseanalyticss3KinesisAnalytics0C0663D4": { + "Type": "AWS::KinesisAnalytics::Application", + "Properties": { + "Inputs": [ + { + "InputSchema": { + "RecordColumns": [ + { + "Mapping": "$.ticker_symbol", + "Name": "ticker_symbol", + "SqlType": "VARCHAR(4)" + }, + { + "Mapping": "$.sector", + "Name": "sector", + "SqlType": "VARCHAR(16)" + }, + { + "Mapping": "$.change", + "Name": "change", + "SqlType": "REAL" + }, + { + "Mapping": "$.price", + "Name": "price", + "SqlType": "REAL" + } + ], + "RecordEncoding": "UTF-8", + "RecordFormat": { + "RecordFormatType": "JSON" + } + }, + "KinesisFirehoseInput": { + "ResourceARN": { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehose9D683007", + "Arn" + ] + }, + "RoleARN": { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisAnalyticsRoleB66D81BD", + "Arn" + ] + } + }, + "NamePrefix": "SOURCE_SQL_STREAM" + } + ] + }, + "DependsOn": [ + "testkinesisfirehoseanalyticss3KinesisAnalyticsPolicy9CAEA1BF" + ] + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store." + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.customLoggingBucket.ts b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.customLoggingBucket.ts new file mode 100644 index 000000000..19627385d --- /dev/null +++ b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.customLoggingBucket.ts @@ -0,0 +1,64 @@ +/** + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +/// !cdk-integ * +import { App, Stack, RemovalPolicy } from "@aws-cdk/core"; +import { BucketEncryption } from "@aws-cdk/aws-s3"; +import { KinesisFirehoseToAnalyticsAndS3 } from "../lib"; +import { generateIntegStackName } from '@aws-solutions-constructs/core'; + +const app = new App(); + +// Empty arguments +const stack = new Stack(app, generateIntegStackName(__filename)); + +new KinesisFirehoseToAnalyticsAndS3(stack, 'test-kinesisfirehose-analytics-s3', { + kinesisAnalyticsProps: { + inputs: [{ + inputSchema: { + recordColumns: [{ + name: 'ticker_symbol', + sqlType: 'VARCHAR(4)', + mapping: '$.ticker_symbol' + }, { + name: 'sector', + sqlType: 'VARCHAR(16)', + mapping: '$.sector' + }, { + name: 'change', + sqlType: 'REAL', + mapping: '$.change' + }, { + name: 'price', + sqlType: 'REAL', + mapping: '$.price' + }], + recordFormat: { + recordFormatType: 'JSON' + }, + recordEncoding: 'UTF-8' + }, + namePrefix: 'SOURCE_SQL_STREAM' + }] + }, + bucketProps: { + removalPolicy: RemovalPolicy.DESTROY, + }, + loggingBucketProps: { + removalPolicy: RemovalPolicy.DESTROY, + bucketName: 'custom-logging-bucket', + encryption: BucketEncryption.S3_MANAGED, + versioned: true + } +}); +app.synth(); \ No newline at end of file diff --git a/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.noLoggingBucket.expected.json b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.noLoggingBucket.expected.json new file mode 100644 index 000000000..ddb278cb9 --- /dev/null +++ b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.noLoggingBucket.expected.json @@ -0,0 +1,411 @@ +{ + "Resources": { + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56": { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketEncryption": { + "ServerSideEncryptionConfiguration": [ + { + "ServerSideEncryptionByDefault": { + "SSEAlgorithm": "AES256" + } + } + ] + }, + "LifecycleConfiguration": { + "Rules": [ + { + "NoncurrentVersionTransitions": [ + { + "StorageClass": "GLACIER", + "TransitionInDays": 90 + } + ], + "Status": "Enabled" + } + ] + }, + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": true, + "BlockPublicPolicy": true, + "IgnorePublicAcls": true, + "RestrictPublicBuckets": true + }, + "VersioningConfiguration": { + "Status": "Enabled" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketPolicy55E9C081": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": "*", + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + } + }, + "Effect": "Deny", + "Principal": { + "AWS": "*" + }, + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56", + "Arn" + ] + }, + "/*" + ] + ] + }, + { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56", + "Arn" + ] + } + ], + "Sid": "HttpsOnly" + } + ], + "Version": "2012-10-17" + } + } + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupA3650D3F": { + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain", + "Metadata": { + "cfn_nag": { + "rules_to_suppress": [ + { + "id": "W86", + "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely" + }, + { + "id": "W84", + "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)" + } + ] + } + } + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupfirehoselogstream508B1DD2": { + "Type": "AWS::Logs::LogStream", + "Properties": { + "LogGroupName": { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupA3650D3F" + } + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehoseRoleD36244BC": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehosePolicy9CE17DCF": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": "logs:PutLogEvents", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:", + { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupA3650D3F" + }, + ":log-stream:", + { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupfirehoselogstream508B1DD2" + } + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehosePolicy9CE17DCF", + "Roles": [ + { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehoseRoleD36244BC" + } + ] + } + }, + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehose9D683007": { + "Type": "AWS::KinesisFirehose::DeliveryStream", + "Properties": { + "DeliveryStreamEncryptionConfigurationInput": { + "KeyType": "AWS_OWNED_CMK" + }, + "ExtendedS3DestinationConfiguration": { + "BucketARN": { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3S3BucketA83D2E56", + "Arn" + ] + }, + "BufferingHints": { + "IntervalInSeconds": 300, + "SizeInMBs": 5 + }, + "CloudWatchLoggingOptions": { + "Enabled": true, + "LogGroupName": { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupA3650D3F" + }, + "LogStreamName": { + "Ref": "testkinesisfirehoseanalyticss3KinesisFirehoseToS3firehoseloggroupfirehoselogstream508B1DD2" + } + }, + "CompressionFormat": "GZIP", + "EncryptionConfiguration": { + "KMSEncryptionConfig": { + "AWSKMSKeyARN": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":kms:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":alias/aws/s3" + ] + ] + } + } + }, + "RoleARN": { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehoseRoleD36244BC", + "Arn" + ] + } + } + } + }, + "testkinesisfirehoseanalyticss3KinesisAnalyticsRoleB66D81BD": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "kinesisanalytics.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "testkinesisfirehoseanalyticss3KinesisAnalyticsPolicy9CAEA1BF": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "firehose:DescribeDeliveryStream", + "firehose:Get*" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehose9D683007", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "testkinesisfirehoseanalyticss3KinesisAnalyticsPolicy9CAEA1BF", + "Roles": [ + { + "Ref": "testkinesisfirehoseanalyticss3KinesisAnalyticsRoleB66D81BD" + } + ] + } + }, + "testkinesisfirehoseanalyticss3KinesisAnalytics0C0663D4": { + "Type": "AWS::KinesisAnalytics::Application", + "Properties": { + "Inputs": [ + { + "InputSchema": { + "RecordColumns": [ + { + "Mapping": "$.ticker_symbol", + "Name": "ticker_symbol", + "SqlType": "VARCHAR(4)" + }, + { + "Mapping": "$.sector", + "Name": "sector", + "SqlType": "VARCHAR(16)" + }, + { + "Mapping": "$.change", + "Name": "change", + "SqlType": "REAL" + }, + { + "Mapping": "$.price", + "Name": "price", + "SqlType": "REAL" + } + ], + "RecordEncoding": "UTF-8", + "RecordFormat": { + "RecordFormatType": "JSON" + } + }, + "KinesisFirehoseInput": { + "ResourceARN": { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisFirehoseToS3KinesisFirehose9D683007", + "Arn" + ] + }, + "RoleARN": { + "Fn::GetAtt": [ + "testkinesisfirehoseanalyticss3KinesisAnalyticsRoleB66D81BD", + "Arn" + ] + } + }, + "NamePrefix": "SOURCE_SQL_STREAM" + } + ] + }, + "DependsOn": [ + "testkinesisfirehoseanalyticss3KinesisAnalyticsPolicy9CAEA1BF" + ] + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store." + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.noLoggingBucket.ts b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.noLoggingBucket.ts new file mode 100644 index 000000000..7fe8cc65a --- /dev/null +++ b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/integ.noLoggingBucket.ts @@ -0,0 +1,58 @@ +/** + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +/// !cdk-integ * +import { App, Stack, RemovalPolicy } from "@aws-cdk/core"; +import { KinesisFirehoseToAnalyticsAndS3 } from "../lib"; +import { generateIntegStackName } from '@aws-solutions-constructs/core'; + +const app = new App(); + +// Empty arguments +const stack = new Stack(app, generateIntegStackName(__filename)); + +new KinesisFirehoseToAnalyticsAndS3(stack, 'test-kinesisfirehose-analytics-s3', { + kinesisAnalyticsProps: { + inputs: [{ + inputSchema: { + recordColumns: [{ + name: 'ticker_symbol', + sqlType: 'VARCHAR(4)', + mapping: '$.ticker_symbol' + }, { + name: 'sector', + sqlType: 'VARCHAR(16)', + mapping: '$.sector' + }, { + name: 'change', + sqlType: 'REAL', + mapping: '$.change' + }, { + name: 'price', + sqlType: 'REAL', + mapping: '$.price' + }], + recordFormat: { + recordFormatType: 'JSON' + }, + recordEncoding: 'UTF-8' + }, + namePrefix: 'SOURCE_SQL_STREAM' + }] + }, + bucketProps: { + removalPolicy: RemovalPolicy.DESTROY, + }, + logS3AccessLogs: false +}); +app.synth(); \ No newline at end of file diff --git a/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/test.kinesisfirehose-analytics-s3.test.ts b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/test.kinesisfirehose-analytics-s3.test.ts index 442d0cd04..6c8efa1cc 100644 --- a/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/test.kinesisfirehose-analytics-s3.test.ts +++ b/source/patterns/@aws-solutions-constructs/aws-kinesisfirehose-s3-and-kinesisanalytics/test/test.kinesisfirehose-analytics-s3.test.ts @@ -98,7 +98,8 @@ test('test kinesisFirehose override ', () => { IntervalInSeconds: 600, SizeInMBs: 55 } - }}); + } + }); }); // -------------------------------------------------------------- @@ -121,4 +122,93 @@ test("Test bad call with existingBucket and bucketProps", () => { }; // Assertion expect(app).toThrowError(); +}); + +// -------------------------------------------------------------- +// s3 bucket with bucket, loggingBucket, and auto delete objects +// -------------------------------------------------------------- +test('s3 bucket with bucket, loggingBucket, and auto delete objects', () => { + const stack = new Stack(); + + new KinesisFirehoseToAnalyticsAndS3(stack, 'kinsisfirehose-s3-analytics', { + kinesisAnalyticsProps: { + inputs: [{ + inputSchema: { + recordColumns: [{ + name: 'ts', + sqlType: 'TIMESTAMP', + mapping: '$.timestamp' + }, { + name: 'trip_id', + sqlType: 'VARCHAR(64)', + mapping: '$.trip_id' + }], + recordFormat: { + recordFormatType: 'JSON' + }, + recordEncoding: 'UTF-8' + }, + namePrefix: 'SOURCE_SQL_STREAM' + }] + }, + loggingBucketProps: { + removalPolicy: RemovalPolicy.DESTROY, + autoDeleteObjects: true + } + }); + + expect(stack).toHaveResource("AWS::S3::Bucket", { + AccessControl: "LogDeliveryWrite" + }); + + expect(stack).toHaveResource("Custom::S3AutoDeleteObjects", { + ServiceToken: { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + BucketName: { + Ref: "kinsisfirehoses3analyticsKinesisFirehoseToS3S3LoggingBucket6EE455EF" + } + }); +}); + +// -------------------------------------------------------------- +// s3 bucket with one content bucket and no logging bucket +// -------------------------------------------------------------- +test('s3 bucket with one content bucket and no logging bucket', () => { + const stack = new Stack(); + + new KinesisFirehoseToAnalyticsAndS3(stack, 'kinsisfirehose-s3-analytics', { + kinesisAnalyticsProps: { + inputs: [{ + inputSchema: { + recordColumns: [{ + name: 'ts', + sqlType: 'TIMESTAMP', + mapping: '$.timestamp' + }, { + name: 'trip_id', + sqlType: 'VARCHAR(64)', + mapping: '$.trip_id' + }], + recordFormat: { + recordFormatType: 'JSON' + }, + recordEncoding: 'UTF-8' + }, + namePrefix: 'SOURCE_SQL_STREAM' + }] + }, + kinesisFirehoseProps: { + deliveryStreamType: 'KinesisStreamAsSource' + }, + bucketProps: { + removalPolicy: RemovalPolicy.DESTROY, + }, + logS3AccessLogs: false + }); + + expect(stack).toCountResources("AWS::S3::Bucket", 1); }); \ No newline at end of file