From 7cd84458db5dbc5e60f4a2ea442758312817f3a2 Mon Sep 17 00:00:00 2001 From: Otavio Macedo <288203+otaviomacedo@users.noreply.github.com> Date: Wed, 19 Apr 2023 11:05:51 +0100 Subject: [PATCH] chore(kinesisfirehose): small improvements to the README. (#25176) The main change is in the paragraph about KMS keys. AWS KMS is replacing the term _customer master key (CMK)_ (which the text was incorrectly referring to as _customer managed key_) with _AWS KMS key_ and _KMS key_. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../aws-kinesisfirehose-alpha/README.md | 82 ++++++++++--------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/packages/@aws-cdk/aws-kinesisfirehose-alpha/README.md b/packages/@aws-cdk/aws-kinesisfirehose-alpha/README.md index 8f24b1438f1ac..564844bcfdb2b 100644 --- a/packages/@aws-cdk/aws-kinesisfirehose-alpha/README.md +++ b/packages/@aws-cdk/aws-kinesisfirehose-alpha/README.md @@ -1,4 +1,5 @@ # Amazon Kinesis Data Firehose Construct Library + --- @@ -76,8 +77,8 @@ new firehose.DeliveryStream(this, 'Delivery Stream', { ### Direct Put -Data must be provided via "direct put", ie., by using a `PutRecord` or `PutRecordBatch` API call. There are a number of ways of doing -so, such as: +Data must be provided via "direct put", ie., by using a `PutRecord` or +`PutRecordBatch` API call. There are a number of ways of doing so, such as: - Kinesis Agent: a standalone Java application that monitors and delivers files while handling file rotation, checkpointing, and retries. See: [Writing to Kinesis Data Firehose Using Kinesis Agent](https://docs.aws.amazon.com/firehose/latest/dev/writing-with-agents.html) @@ -111,9 +112,9 @@ new firehose.DeliveryStream(this, 'Delivery Stream', { }); ``` -The S3 destination also supports custom dynamic prefixes. `prefix` will be used for files -successfully delivered to S3. `errorOutputPrefix` will be added to failed records before -writing them to S3. +The S3 destination also supports custom dynamic prefixes. `dataOutputPrefix` +will be used for files successfully delivered to S3. `errorOutputPrefix` will be added to +failed records before writing them to S3. ```ts declare const bucket: s3.Bucket; @@ -123,7 +124,8 @@ const s3Destination = new destinations.S3Bucket(bucket, { }); ``` -See: [Custom S3 Prefixes](https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html) in the *Kinesis Data Firehose Developer Guide*. +See: [Custom S3 Prefixes](https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html) +in the *Kinesis Data Firehose Developer Guide*. ## Server-side Encryption @@ -132,33 +134,34 @@ sent to delivery stream when it is stored at rest. This means that data is encry before being written to the service's internal storage layer and decrypted after it is received from the internal storage layer. The service manages keys and cryptographic operations so that sources and destinations do not need to, as the data is encrypted and -decrypted at the boundaries of the service (ie., before the data is delivered to a +decrypted at the boundaries of the service (i.e., before the data is delivered to a destination). By default, delivery streams do not have SSE enabled. -The Key Management Service (KMS) Customer Managed Key (CMK) used for SSE can either be -AWS-owned or customer-managed. AWS-owned CMKs are keys that an AWS service (in this case -Kinesis Data Firehose) owns and manages for use in multiple AWS accounts. As a customer, -you cannot view, use, track, or manage these keys, and you are not charged for their -use. On the other hand, customer-managed CMKs are keys that are created and owned within -your account and managed entirely by you. As a customer, you are responsible for managing -access, rotation, aliases, and deletion for these keys, and you are changed for their -use. See: [Customer master keys](https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#master_keys) +The Key Management Service keys (KMS keys) used for SSE can either be AWS-owned or +customer-managed. AWS-owned KMS keys are created, owned and managed by AWS for use in +multiple AWS accounts. As a customer, you cannot view, use, track, or manage these keys, +and you are not charged for their use. On the other hand, customer-managed KMS keys are +created and owned within your account and managed entirely by you. As a customer, you are +responsible for managing access, rotation, aliases, and deletion for these keys, and you +are changed for their use. + +See: [AWS KMS keys](https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#kms_keys) in the *KMS Developer Guide*. ```ts declare const destination: firehose.IDestination; -// SSE with an AWS-owned CMK +// SSE with an AWS-owned key new firehose.DeliveryStream(this, 'Delivery Stream AWS Owned', { encryption: firehose.StreamEncryption.AWS_OWNED, destinations: [destination], }); -// SSE with an customer-managed CMK that is created automatically by the CDK +// SSE with an customer-managed key that is created automatically by the CDK new firehose.DeliveryStream(this, 'Delivery Stream Implicit Customer Managed', { encryption: firehose.StreamEncryption.CUSTOMER_MANAGED, destinations: [destination], }); -// SSE with an customer-managed CMK that is explicitly specified +// SSE with an customer-managed key that is explicitly specified declare const key: kms.Key; new firehose.DeliveryStream(this, 'Delivery Stream Explicit Customer Managed', { encryptionKey: key, @@ -166,8 +169,8 @@ new firehose.DeliveryStream(this, 'Delivery Stream Explicit Customer Managed', { }); ``` -See: [Data Protection](https://docs.aws.amazon.com/firehose/latest/dev/encryption.html) in -the *Kinesis Data Firehose Developer Guide*. +See: [Data Protection](https://docs.aws.amazon.com/firehose/latest/dev/encryption.html) +in the *Kinesis Data Firehose Developer Guide*. ## Monitoring @@ -180,8 +183,8 @@ Kinesis Data Firehose will send logs to CloudWatch when data transformation or d delivery fails. The CDK will enable logging by default and create a CloudWatch LogGroup and LogStream for your Delivery Stream. -You can provide a specific log group to specify where the CDK will create the log streams -where log events will be sent: +When you create a destination, you can specify a log group. In this log group, The CDK +will create log streams where log events will be sent: ```ts import * as logs from 'aws-cdk-lib/aws-logs'; @@ -192,7 +195,6 @@ const destination = new destinations.S3Bucket(bucket, { logGroup: logGroup, }); -declare const destination: firehose.IDestination; new firehose.DeliveryStream(this, 'Delivery Stream', { destinations: [destination], }); @@ -232,6 +234,7 @@ are pre-populated with the correct dimensions for the delivery stream. ```ts import * as cloudwatch from 'aws-cdk-lib/aws-cloudwatch'; + declare const deliveryStream: firehose.DeliveryStream; // Alarm that triggers when the per-second average of incoming bytes exceeds 90% of the current service limit @@ -256,7 +259,7 @@ in the *Kinesis Data Firehose Developer Guide*. ## Compression Your data can automatically be compressed when it is delivered to S3 as either a final or -an intermediary/backup destination. Supported compression formats are: gzip, Snappy, +an intermediary/backup destination. Supported compression formats are: gzip, Snappy, Hadoop-compatible Snappy, and ZIP, except for Redshift destinations, where Snappy (regardless of Hadoop-compatibility) and ZIP are not supported. By default, data is delivered to S3 without compression. @@ -298,12 +301,13 @@ in the *Kinesis Data Firehose Developer Guide*. ## Destination Encryption -Your data can be automatically encrypted when it is delivered to S3 as a final or -an intermediary/backup destination. Kinesis Data Firehose supports Amazon S3 server-side -encryption with AWS Key Management Service (AWS KMS) for encrypting delivered data -in Amazon S3. You can choose to not encrypt the data or to encrypt with a key from -the list of AWS KMS keys that you own. For more information, see [Protecting Data -Using Server-Side Encryption with AWS KMS–Managed Keys (SSE-KMS)](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). Data is not encrypted by default. +Your data can be automatically encrypted when it is delivered to S3 as a final or an +intermediary/backup destination. Kinesis Data Firehose supports Amazon S3 server-side +encryption with AWS Key Management Service (AWS KMS) for encrypting delivered data in +Amazon S3. You can choose to not encrypt the data or to encrypt with a key from the list +of AWS KMS keys that you own. For more information, +see [Protecting Data Using Server-Side Encryption with AWS KMS–Managed Keys (SSE-KMS)](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). +Data is not encrypted by default. ```ts declare const bucket: s3.Bucket; @@ -318,10 +322,10 @@ new firehose.DeliveryStream(this, 'Delivery Stream', { ## Backup -A delivery stream can be configured to backup data to S3 that it attempted to deliver to +A delivery stream can be configured to back up data to S3 that it attempted to deliver to the configured destination. Backed up data can be all the data that the delivery stream attempted to deliver or just data that it failed to deliver (Redshift and S3 destinations -can only backup all data). CDK can create a new S3 bucket where it will back up data or +can only back up all data). CDK can create a new S3 bucket where it will back up data, or you can provide a bucket where data will be backed up. You can also provide a prefix under which your backed-up data will be placed within the bucket. By default, source data is not backed up to S3. @@ -387,10 +391,11 @@ result that contains records in a specific format, including the following field - `data` -- the transformed data, Base64-encoded. The data is buffered up to 1 minute and up to 3 MiB by default before being sent to the -function, but can be configured using `bufferInterval` and `bufferSize` in the processor -configuration (see: [Buffering](#buffering)). If the function invocation fails due to a -network timeout or because of hitting an invocation limit, the invocation is retried 3 -times by default, but can be configured using `retries` in the processor configuration. +function, but can be configured using `bufferInterval` and `bufferSize` +in the processor configuration (see: [Buffering](#buffering)). If the function invocation +fails due to a network timeout or because of hitting an invocation limit, the invocation +is retried 3 times by default, but can be configured using `retries` in the processor +configuration. ```ts // Provide a Lambda function that will transform records before delivery, with custom @@ -463,7 +468,7 @@ in the *Kinesis Data Firehose Developer Guide*. IAM roles, users or groups which need to be able to work with delivery streams should be granted IAM permissions. -Any object that implements the `IGrantable` interface (ie., has an associated principal) +Any object that implements the `IGrantable` interface (i.e., has an associated principal) can be granted permissions to a delivery stream by calling: - `grantPutRecords(principal)` - grants the principal the ability to put records onto the @@ -481,7 +486,8 @@ declare const deliveryStream: firehose.DeliveryStream; deliveryStream.grantPutRecords(lambdaRole); ``` -The following write permissions are provided to a service principal by the `grantPutRecords()` method: +The following write permissions are provided to a service principal by the +`grantPutRecords()` method: - `firehose:PutRecord` - `firehose:PutRecordBatch`