Migrate configuration from MP repository

README.md

@@ -6,20 +6,21 @@
 
 ```hcl
 
-module "template" {
+module "example" {
 
-  source = "github.com/ministryofjustice/modernisation-platform-terraform-module-template"
-
-  tags             = local.tags
-  application_name = local.application_name
+  source                     = "github.com/ministryofjustice/modernisation-platform-terraform-aws-data-firehose"
+  cloudwatch_log_group_names = ["example-1", "example-2", "example-3"]
+  destination_bucket_arn     = aws_s3_bucket.example.arn
+  tags                       = local.tags
 
 }
 
 ```
-<!--- BEGIN_TF_DOCS --->
 
+This module creates an [AWS Data Stream](https://aws.amazon.com/kinesis/data-streams/) to be used by a set of AWS CloudWatch Log Groups.
+Data is streamed from the Log Groups to a target S3 bucket using a Cloudwatch Log Subscription Filter.
 
-<!--- END_TF_DOCS --->
+Included in this module are the necessary IAM policy documents and roles for these actions, as well as a KMS key to encrypt the Data Stream.
 
 ## Looking for issues?
 If you're looking to raise an issue with this module, please create a new issue in the [Modernisation Platform repository](https://github.com/ministryofjustice/modernisation-platform/issues).
@@ -29,27 +30,52 @@ If you're looking to raise an issue with this module, please create a new issue
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.1 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.0 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.0 |
+| <a name="requirement_random"></a> [random](#requirement\_random) | ~> 3.4 |
 
 ## Providers
 
-No providers.
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.0 |
+| <a name="provider_random"></a> [random](#provider\_random) | ~> 3.4 |
 
 ## Modules
 
 No modules.
 
 ## Resources
 
-No resources.
+| Name | Type |
+|------|------|
+| [aws_cloudwatch_log_group.kinesis](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
+| [aws_cloudwatch_log_subscription_filter.cloudwatch-to-firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_subscription_filter) | resource |
+| [aws_iam_policy.cloudwatch-to-firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.firehose-to-s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_policy_attachment.cloudwatch-to-firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource |
+| [aws_iam_policy_attachment.firehose-to-s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource |
+| [aws_iam_role.cloudwatch-to-firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role.firehose-to-s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_kinesis_firehose_delivery_stream.firehose-to-s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource |
+| [aws_kms_alias.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_alias) | resource |
+| [aws_kms_key.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
+| [random_id.name](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_iam_policy_document.cloudwatch-logs-role-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.cloudwatch-logs-trust-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.firehose-key-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.firehose-role-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.firehose-trust-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_application_name"></a> [application\_name](#input\_application\_name) | Name of application | `string` | n/a | yes |
-| <a name="input_tags"></a> [tags](#input\_tags) | Common tags to be used by all resources | `map(string)` | n/a | yes |
+| <a name="input_cloudwatch_filter_pattern"></a> [cloudwatch\_filter\_pattern](#input\_cloudwatch\_filter\_pattern) | A valid CloudWatch Logs filter pattern for subscribing to a filtered stream of log events. | `string` | `""` | no |
+| <a name="input_cloudwatch_log_group_names"></a> [cloudwatch\_log\_group\_names](#input\_cloudwatch\_log\_group\_names) | List of CloudWatch Log Group names to stream logs from. | `list(string)` | n/a | yes |
+| <a name="input_destination_bucket_arn"></a> [destination\_bucket\_arn](#input\_destination\_bucket\_arn) | ARN of the bucket for CloudWatch filters. | `string` | n/a | yes |
+| <a name="input_tags"></a> [tags](#input\_tags) | Map of tags to be applied to resources. | `map(string)` | n/a | yes |
 
 ## Outputs
 

data.tf

@@ -0,0 +1,132 @@
+data "aws_caller_identity" "current" {}
+
+data "aws_iam_policy_document" "cloudwatch-logs-trust-policy" {
+  version = "2012-10-17"
+
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["logs.eu-west-2.amazonaws.com", ]
+    }
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:SourceArn"
+      values   = ["arn:aws:logs:eu-west-2:${data.aws_caller_identity.current.account_id}:*"]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "cloudwatch-logs-role-policy" {
+  version = "2012-10-17"
+
+  statement {
+    sid    = "FirehoseToDeliveryStream"
+    effect = "Allow"
+    actions = [
+      "firehose:PutRecord",
+      "firehose:PutRecordBatch"
+    ]
+    resources = [
+      aws_kinesis_firehose_delivery_stream.firehose-to-s3.arn
+    ]
+  }
+}
+
+data "aws_iam_policy_document" "firehose-trust-policy" {
+  version = "2012-10-17"
+
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["firehose.amazonaws.com", ]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "firehose-role-policy" {
+  version = "2012-10-17"
+
+  statement {
+    sid    = "FirehoseToS3"
+    effect = "Allow"
+    actions = [
+      "s3:GetObject",
+      "s3:PutObject",
+      "s3:PutObjectAcl"
+    ]
+    resources = [
+      var.destination_bucket_arn,
+      "${var.destination_bucket_arn}/*"
+    ]
+  }
+  statement {
+    sid    = "FirehoseUseKMS"
+    effect = "Allow"
+    actions = [
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:GenerateDataKey",
+      "kms:DescribeKey"
+    ]
+    resources = [
+      aws_kms_key.firehose.arn
+    ]
+  }
+  statement {
+    sid    = "FirehosePutLogs"
+    effect = "Allow"
+    actions = [
+      "logs:PutLogEvents"
+    ]
+    resources = [
+      "${aws_cloudwatch_log_group.kinesis.arn}:log-stream:DestinationDelivery"
+    ]
+  }
+}
+
+data "aws_iam_policy_document" "firehose-key-policy" {
+  # checkov:skip=CKV_AWS_109: Policy appropriately secure
+  # checkov:skip=CKV_AWS_111
+  # checkov:skip=CKV_AWS_356
+  statement {
+    sid    = "KeyAdministration"
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
+    }
+
+    actions   = ["kms:*"]
+    resources = ["*"]
+  }
+
+  statement {
+    sid    = "AllowFirehoseRole"
+    effect = "Allow"
+
+    principals {
+      identifiers = [
+        aws_iam_role.cloudwatch-to-firehose.arn,
+        "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"
+      ]
+      type = "AWS"
+    }
+
+    actions = [
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:DescribeKey"
+    ]
+    resources = ["*"]
+  }
+}

main.tf

@@ -0,0 +1,102 @@
+resource "random_id" "name" {
+  byte_length = 4
+}
+
+resource "aws_kms_key" "firehose" {
+  # checkov:skip=CKV_AWS_7
+  description             = "KMS key for Firehose delivery streams"
+  deletion_window_in_days = 0
+  policy                  = data.aws_iam_policy_document.firehose-key-policy.json
+  tags                    = var.tags
+}
+
+resource "aws_kms_alias" "firehose" {
+  name_prefix   = "alias/firehose-log-delivery-${random_id.name.hex}"
+  target_key_id = aws_kms_key.firehose.id
+}
+
+resource "aws_iam_role" "firehose-to-s3" {
+  assume_role_policy = data.aws_iam_policy_document.firehose-trust-policy.json
+  name_prefix        = "firehose-to-s3"
+  tags               = var.tags
+}
+
+resource "aws_iam_policy" "firehose-to-s3" {
+  name_prefix = "firehose-to-s3"
+  policy      = data.aws_iam_policy_document.firehose-role-policy.json
+  tags        = var.tags
+}
+
+resource "aws_iam_policy_attachment" "firehose-to-s3" {
+  name       = "${aws_iam_role.firehose-to-s3.name}-policy"
+  policy_arn = aws_iam_policy.firehose-to-s3.arn
+  roles      = [aws_iam_role.firehose-to-s3.name]
+}
+
+resource "aws_iam_role" "cloudwatch-to-firehose" {
+  assume_role_policy = data.aws_iam_policy_document.cloudwatch-logs-trust-policy.json
+  name_prefix        = "cloudwatch-to-firehose"
+  tags               = var.tags
+}
+
+resource "aws_iam_policy" "cloudwatch-to-firehose" {
+  name_prefix = "cloudwatch-to-firehose"
+  policy      = data.aws_iam_policy_document.cloudwatch-logs-role-policy.json
+  tags        = var.tags
+}
+
+resource "aws_iam_policy_attachment" "cloudwatch-to-firehose" {
+  name       = "${aws_iam_role.cloudwatch-to-firehose.name}-policy"
+  policy_arn = aws_iam_policy.cloudwatch-to-firehose.arn
+  roles      = [aws_iam_role.cloudwatch-to-firehose.name]
+}
+
+resource "aws_kinesis_firehose_delivery_stream" "firehose-to-s3" {
+  destination = "extended_s3"
+  name        = "cloudwatch-to-s3-${random_id.name.hex}"
+
+  extended_s3_configuration {
+    bucket_arn          = var.destination_bucket_arn
+    buffering_size      = 64
+    buffering_interval  = 60
+    compression_format  = "GZIP"
+    role_arn            = aws_iam_role.firehose-to-s3.arn
+    prefix              = "logs/!{timestamp:yyyy/MM/dd}/"
+    error_output_prefix = "errors/!{firehose:error-output-type}/!{timestamp:yyyy/MM/dd}/"
+
+    cloudwatch_logging_options {
+      enabled         = true
+      log_group_name  = aws_cloudwatch_log_group.kinesis.name
+      log_stream_name = "DestinationDelivery"
+    }
+
+    dynamic_partitioning_configuration {
+      enabled = false
+    }
+  }
+
+  server_side_encryption {
+    enabled  = true
+    key_type = "CUSTOMER_MANAGED_CMK"
+    key_arn  = aws_kms_key.firehose.arn
+  }
+
+  tags = var.tags
+}
+
+resource "aws_cloudwatch_log_group" "kinesis" {
+  #  checkov:skip=CKV_AWS_338:Short life error logs don't need long term retention
+  #  checkov:skip=CKV_AWS_158:Default log encryption OK for short life error logs
+  name              = "/aws/kinesisfirehose/cloudwatch-to-s3-${random_id.name.hex}"
+  retention_in_days = 14
+  tags              = var.tags
+}
+
+resource "aws_cloudwatch_log_subscription_filter" "cloudwatch-to-firehose" {
+  for_each        = toset(var.cloudwatch_log_group_names)
+  destination_arn = aws_kinesis_firehose_delivery_stream.firehose-to-s3.arn
+  filter_pattern  = var.cloudwatch_filter_pattern
+  log_group_name  = each.key
+  name            = "firehose-delivery-${each.key}-${random_id.name.hex}"
+  role_arn        = aws_iam_role.cloudwatch-to-firehose.arn
+}

variables.tf

@@ -1,8 +1,20 @@
-variable "tags" {
-  type        = map(string)
-  description = "Common tags to be used by all resources"
+variable "cloudwatch_log_group_names" {
+  type        = list(string)
+  description = "List of CloudWatch Log Group names to stream logs from."
+}
+
+variable "destination_bucket_arn" {
+  type        = string
+  description = "ARN of the bucket for CloudWatch filters."
 }
-variable "application_name" {
+
+variable "cloudwatch_filter_pattern" {
   type        = string
-  description = "Name of application"
+  description = "A valid CloudWatch Logs filter pattern for subscribing to a filtered stream of log events."
+  default     = ""
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "Map of tags to be applied to resources."
 }

versions.tf

@@ -1,9 +1,13 @@
 terraform {
+  required_version = "~> 1.0"
   required_providers {
     aws = {
-      version = "~> 5.0"
       source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.4"
     }
   }
-  required_version = ">= 1.0.1"
 }