Merge pull request #7705 from ministryofjustice/feature/7607-add-xsia…

…m-bucket-to-core-logging

Stream logs from CloudWatch to S3 bucket in `core-logging` account

terraform/environments/core-logging/cortex.tf

@@ -0,0 +1,145 @@
+data "aws_iam_policy_document" "logging-bucket" {
+  statement {
+    sid     = "EnforceTLSv12orHigher"
+    effect  = "Deny"
+    actions = ["s3:*"]
+    resources = [
+      aws_s3_bucket.logging.arn,
+      "${aws_s3_bucket.logging.arn}/*"
+    ]
+    principals {
+      identifiers = ["*"]
+      type        = "AWS"
+    }
+    condition {
+      test     = "NumericLessThan"
+      variable = "s3:TlsVersion"
+      values   = [1.2]
+    }
+  }
+  statement {
+    sid    = "AllowFirehosePutObject"
+    effect = "Allow"
+    principals {
+      type        = "Service"
+      identifiers = ["firehose.amazonaws.com"]
+    }
+    actions = [
+      "s3:AbortMultipartUpload",
+      "s3:GetBucketLocation",
+      "s3:GetObject",
+      "s3:ListBucket",
+      "s3:ListBucketMultipartUploads",
+      "s3:PutObject"
+    ]
+    resources = [
+      aws_s3_bucket.logging.arn,
+      "${aws_s3_bucket.logging.arn}/*"
+    ]
+    condition {
+      test     = "ForAnyValue:StringLike"
+      variable = "aws:PrincipalOrgPaths"
+      values   = ["${data.aws_organizations_organization.root_account.id}/*/${local.environment_management.modernisation_platform_organisation_unit_id}/*"]
+    }
+    condition {
+      test     = "ArnLike"
+      variable = "aws:SourceArn"
+      values   = ["arn:aws:firehose:*:*:*"]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "logging-sqs" {
+  statement {
+    sid    = "AllowSendMessage"
+    effect = "Allow"
+    principals {
+      type        = "Service"
+      identifiers = ["s3.amazonaws.com"]
+    }
+    actions = ["sqs:SendMessage"]
+    resources = [
+      aws_sqs_queue.logging.arn
+    ]
+    condition {
+      test     = "ArnEquals"
+      variable = "aws:SourceArn"
+      values   = [aws_s3_bucket.logging.arn]
+    }
+  }
+}
+
+resource "aws_s3_bucket" "logging" {
+  #  checkov:skip=CKV_AWS_18: Access logs not presently required
+  #  checkov:skip=CKV_AWS_21: Versioning of log objects not required
+  #  checkov:skip=CKV_AWS_144:Replication of log objects not required
+  bucket_prefix = terraform.workspace
+  tags          = local.tags
+}
+
+resource "aws_s3_bucket_acl" "bucket_acl" {
+  bucket = aws_s3_bucket.logging.id
+  acl    = "private"
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "example" {
+  bucket = aws_s3_bucket.logging.id
+
+  rule {
+    abort_incomplete_multipart_upload {
+      days_after_initiation = 7
+    }
+    id = "rule-1"
+    filter {}
+    expiration {
+      days = 14
+    }
+    status = "Enabled"
+  }
+}
+
+resource "aws_s3_bucket_notification" "logging" {
+  bucket = aws_s3_bucket.logging.id
+  queue {
+    queue_arn = aws_sqs_queue.logging.arn
+    events    = ["s3:ObjectCreated:*"] # Events to trigger the notification
+  }
+}
+
+resource "aws_s3_bucket_policy" "logging" {
+  bucket = aws_s3_bucket.logging.id
+  policy = data.aws_iam_policy_document.logging-bucket.json
+}
+
+resource "aws_s3_bucket_public_access_block" "example" {
+  bucket                  = aws_s3_bucket.logging.id
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "logging" {
+  bucket = aws_s3_bucket.logging.id
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "aws:kms"
+    }
+  }
+}
+
+resource "aws_sqs_queue" "logging" {
+  name_prefix                = terraform.workspace
+  delay_seconds              = 0      # The default is 0 but can be up to 15 minutes
+  max_message_size           = 262144 # 256k which is the max size
+  message_retention_seconds  = 345600 # This is 4 days. The max is 14 days
+  sqs_managed_sse_enabled    = true   # Using managed encryption
+  visibility_timeout_seconds = 30     # This is only useful for queues that have multiple subscribers
+  tags                       = local.tags
+}
+
+resource "aws_sqs_queue_policy" "logging" {
+  policy    = data.aws_iam_policy_document.logging-sqs.json
+  queue_url = aws_sqs_queue.logging.url
+}

terraform/environments/core-logging/locals.tf

@@ -2,6 +2,8 @@ data "aws_caller_identity" "modernisation-platform" {
   provider = aws.modernisation-platform
 }
 
+data "aws_organizations_organization" "root_account" {}
+
 locals {
   application_name           = "core-logging"
   environment_management     = jsondecode(data.aws_secretsmanager_secret_version.environment_management.secret_string)

terraform/modules/cloudwatch-firehose/data.tf

@@ -0,0 +1,115 @@
+data "aws_caller_identity" "current" {}
+
+data "aws_iam_policy_document" "cloudwatch-logs-trust-policy" {
+  version = "2012-10-17"
+
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["logs.amazonaws.com", ]
+    }
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:SourceArn"
+      values   = ["arn:aws:logs:region:${data.aws_caller_identity.destination.account_id}:*"]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "cloudwatch-logs-role-policy" {
+  version = "2012-10-17"
+
+  statement {
+    sid    = "FirehoseToDeliveryStream"
+    effect = "Allow"
+    actions = [
+      "firehose:PutRecord"
+    ]
+    resources = [
+      "arn:aws:firehose:*:account-id:deliverystream/delivery-stream-name"
+    ]
+  }
+}
+
+data "aws_iam_policy_document" "firehose-trust-policy" {
+  version = "2012-10-17"
+
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["firehose.amazonaws.com", ]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "firehose-role-policy" {
+  version = "2012-10-17"
+
+  statement {
+    sid    = "FirehoseToS3"
+    effect = "Allow"
+    actions = [
+      "s3:AbortMultipartUpload",
+      "s3:GetBucketLocation",
+      "s3:GetObject",
+      "s3:ListBucket",
+      "s3:ListBucketMultipartUploads",
+      "s3:PutObject"
+    ]
+    resources = [
+      var.destination_bucket_arn,
+      "${var.destination_bucket_arn}/*"
+    ]
+  }
+}
+
+data "aws_iam_policy_document" "firehose-key-policy" {
+  # checkov:skip=CKV_AWS_109: Policy appropriately secure
+  # checkov:skip=CKV_AWS_111
+  # checkov:skip=CKV_AWS_356
+  statement {
+    sid    = "KeyAdministration"
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
+    }
+
+    actions   = ["kms:*"]
+    resources = ["*"]
+  }
+
+  statement {
+    sid    = "AllowFirehose"
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["firehose.amazonaws.com"]
+    }
+
+    actions = [
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:DescribeKey"
+    ]
+
+    resources = ["*"]
+
+    condition {
+      test     = "StringEquals"
+      variable = "kms:ViaService"
+      values   = ["firehose.amazonaws.com"]
+    }
+  }
+}

terraform/modules/cloudwatch-firehose/main.tf

@@ -0,0 +1,104 @@
+resource "random_id" "name" {
+  byte_length = 4
+}
+
+resource "aws_kms_key" "firehose" {
+  # checkov:skip=CKV_AWS_7
+  description             = "KMS key for Firehose delivery streams"
+  deletion_window_in_days = 7
+  policy                  = data.aws_iam_policy_document.firehose-key-policy.json
+  tags                    = var.tags
+}
+
+resource "aws_kms_alias" "firehose" {
+  name          = "firehose-log-delivery"
+  target_key_id = aws_kms_key.firehose.id
+}
+
+resource "aws_iam_role" "firehose-to-s3" {
+  assume_role_policy = data.aws_iam_policy_document.firehose-trust-policy.json
+  name_prefix        = "firehose-to-s3"
+  tags               = var.tags
+}
+
+resource "aws_iam_policy" "firehose-to-s3" {
+  policy = data.aws_iam_policy_document.firehose-role-policy.json
+  tags   = var.tags
+}
+
+resource "aws_iam_policy_attachment" "firehose-to-s3" {
+  name       = "${aws_iam_role.firehose-to-s3.name}-policy"
+  policy_arn = aws_iam_policy.firehose-to-s3.arn
+  roles      = [aws_iam_role.firehose-to-s3.name]
+}
+
+resource "aws_iam_role" "cloudwatch-to-firehose" {
+  assume_role_policy = data.aws_iam_policy_document.cloudwatch-logs-trust-policy.json
+  name_prefix        = "cloudwatch-to-firehose"
+  tags               = var.tags
+}
+
+resource "aws_iam_policy" "cloudwatch-to-firehose" {
+  policy = data.aws_iam_policy_document.cloudwatch-logs-role-policy.json
+  tags   = var.tags
+}
+
+resource "aws_iam_policy_attachment" "cloudwatch-to-firehose" {
+  name       = "${aws_iam_role.cloudwatch-to-firehose.name}-policy"
+  policy_arn = aws_iam_policy.cloudwatch-to-firehose.arn
+  roles      = [aws_iam_role.cloudwatch-to-firehose.name]
+}
+
+resource "aws_kinesis_firehose_delivery_stream" "firehose-to-s3" {
+  destination = "extended_s3"
+  name        = "cloudwatch-to-s3-${random_id.name.hex}"
+
+  extended_s3_configuration {
+    bucket_arn = var.destination_bucket_arn
+    role_arn   = aws_iam_role.firehose-to-s3.arn
+
+    prefix              = "logs/!{partitionKeyFromQuery:logGroupName}/"
+    error_output_prefix = "errors/!{firehose:error-output-type}/!{timestamp:yyyy/MM/dd}/"
+
+    buffer_size     = 64
+    buffer_interval = 60
+
+    dynamic_partitioning_configuration {
+      enabled = true
+    }
+
+    processing_configuration {
+      enabled = true
+
+      processors {
+        type = "MetadataExtraction"
+        parameters {
+          parameter_name  = "JsonParsingEngine"
+          parameter_value = "JQ"
+        }
+        parameters {
+          parameter_name  = "MetadataExtractionQuery"
+          parameter_value = "{logGroupName:.logGroup}"
+        }
+      }
+    }
+
+  }
+
+  server_side_encryption {
+    enabled  = true
+    key_type = "CUSTOMER_MANAGED_CMK"
+    key_arn  = aws_kms_key.firehose.arn
+  }
+
+  tags = var.tags
+}
+
+resource "aws_cloudwatch_log_subscription_filter" "cloudwatch-to-firehose" {
+  for_each        = toset(var.cloudwatch_log_groups)
+  destination_arn = aws_kinesis_firehose_delivery_stream.firehose-to-s3.arn
+  filter_pattern  = "" # Left empty to stream all logs
+  log_group_name  = each.key
+  name            = "firehose-delivery-${each.key}"
+  role_arn        = aws_iam_role.cloudwatch-to-firehose.arn
+}

terraform/modules/cloudwatch-firehose/variables.tf

@@ -0,0 +1,14 @@
+variable "cloudwatch_log_groups" {
+  type        = list(string)
+  description = "List of CloudWatch Log Group names to stream logs from."
+}
+
+variable "destination_bucket_arn" {
+  type        = string
+  description = "ARN of the bucket for CloudWatch filters."
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "Map of tags to be applied to resources."
+}