Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Stream logs to separate buckets #7772

Merged
merged 3 commits into from
Aug 23, 2024
Merged

Stream logs to separate buckets #7772

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
354ba33
stream logs to separate buckets
dms1981 Aug 23, 2024
1f0bbe5
stream logs to separate buckets
dms1981 Aug 23, 2024
381d890
fixed local values to correctly output flat lists
dms1981 Aug 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 4 additions & 9 deletions terraform/environments/core-network-services/locals.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,10 @@ locals {
is-production = substr(terraform.workspace, length(local.application_name), length(terraform.workspace)) == "-production"

# This local allows us to references the key / value pairs held in xsiam_secrets.
xsiam = jsondecode(data.aws_secretsmanager_secret_version.xsiam_secret_arn_version.secret_string)
cloudwatch_log_bucket = data.aws_secretsmanager_secret_version.core_logging_bucket_arn.secret_string
cloudwatch_log_groups = local.is-production ? concat([
aws_cloudwatch_log_group.external_inspection.name,
aws_cloudwatch_log_group.tgw_flowlog_group.name,
module.firewall_logging.cloudwatch_log_group_name],
[for key, value in module.vpc_inspection : value.vpc_cloudwatch_name],
[for key, value in module.vpc_inspection : value.fw_cloudwatch_name]
) : []
xsiam = jsondecode(data.aws_secretsmanager_secret_version.xsiam_secret_arn_version.secret_string)
cloudwatch_log_buckets = jsondecode(data.aws_secretsmanager_secret_version.core_logging_bucket_arns.secret_string)
cloudwatch_generic_log_groups = concat([module.firewall_logging.cloudwatch_log_group_name], [for key, value in module.vpc_inspection : value.fw_cloudwatch_name])
cloudwatch_vpc_flow_log_groups = concat([aws_cloudwatch_log_group.external_inspection.name, aws_cloudwatch_log_group.tgw_flowlog_group.name], [for key, value in module.vpc_inspection : value.vpc_cloudwatch_name])

tags = {
business-unit = "Platforms"
Expand Down
15 changes: 15 additions & 0 deletions terraform/environments/core-network-services/logging.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
module "logging-vpc-flow-logs" {
source = "../../modules/cloudwatch-firehose"
for_each = local.is-production ? { "build" = true } : {}
cloudwatch_log_groups = local.cloudwatch_vpc_flow_log_groups
destination_bucket_arn = local.cloudwatch_log_buckets["vpc-flow-logs"]
tags = local.tags
}

module "logging-generic-logs" {
source = "../../modules/cloudwatch-firehose"
for_each = local.is-production ? { "build" = true } : {}
cloudwatch_log_groups = local.cloudwatch_generic_log_groups
destination_bucket_arn = local.cloudwatch_log_buckets["generic-logs"]
tags = local.tags
}
11 changes: 5 additions & 6 deletions terraform/environments/core-network-services/secrets.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,17 @@ data "aws_secretsmanager_secret_version" "pagerduty_integration_keys" {
secret_id = data.aws_secretsmanager_secret.pagerduty_integration_keys.id
}

data "aws_secretsmanager_secret" "core_logging_bucket_arn" {
# Get the ARNs of the logging buckets in `core-logging`
data "aws_secretsmanager_secret" "core_logging_bucket_arns" {
provider = aws.modernisation-platform
name = "core_logging_bucket_arn"
name = "core_logging_bucket_arns"
}

# Get the ARN of the logging bucket in `core-logging`
data "aws_secretsmanager_secret_version" "core_logging_bucket_arn" {
data "aws_secretsmanager_secret_version" "core_logging_bucket_arns" {
provider = aws.modernisation-platform
secret_id = data.aws_secretsmanager_secret.core_logging_bucket_arn.id
secret_id = data.aws_secretsmanager_secret.core_logging_bucket_arns.id
}


# Data for Firehose Endpoint URL & Key that are held in secrets manager.

data "aws_secretsmanager_secret" "xsiam_secret_arn" {
Expand Down
10 changes: 4 additions & 6 deletions terraform/environments/core-vpc/locals.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,10 @@ locals {
build_firehose = anytrue([local.is-development, local.is-production]) ? true : false

# Secrets used by Firehose resources which we only require for development & production VPCs.
xsiam = jsondecode(data.aws_secretsmanager_secret_version.xsiam_secret_arn_version.secret_string)
cloudwatch_log_bucket = data.aws_secretsmanager_secret_version.core_logging_bucket_arn.secret_string
cloudwatch_log_groups = local.is-production ? concat(
[for env in module.route_53_resolver_logs : env.r53_resolver_log_name],
[for key, value in module.vpc : value.vpc_flow_log]
) : []
xsiam = jsondecode(data.aws_secretsmanager_secret_version.xsiam_secret_arn_version.secret_string)
cloudwatch_log_buckets = jsondecode(data.aws_secretsmanager_secret_version.core_logging_bucket_arns.secret_string)
cloudwatch_r53_resolver_log_groups = local.is-production ? [for env in module.route_53_resolver_logs : env.r53_resolver_log_name] : []
cloudwatch_vpc_flow_log_groups = local.is-production ? [for key, value in module.vpc : value.vpc_flow_log] : []

tags = {
business-unit = "Platforms"
Expand Down
15 changes: 15 additions & 0 deletions terraform/environments/core-vpc/logging.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
module "logging-r53-resolver" {
source = "../../modules/cloudwatch-firehose"
for_each = local.is-production ? { "build" = true } : {}
cloudwatch_log_groups = local.cloudwatch_r53_resolver_log_groups
destination_bucket_arn = local.cloudwatch_log_buckets["r53-resolver-logs"]
tags = local.tags
}

module "logging-vpc-flow-logs" {
source = "../../modules/cloudwatch-firehose"
for_each = local.is-production ? { "build" = true } : {}
cloudwatch_log_groups = local.cloudwatch_vpc_flow_log_groups
destination_bucket_arn = local.cloudwatch_log_buckets["vpc-flow-logs"]
tags = local.tags
}
10 changes: 5 additions & 5 deletions terraform/environments/core-vpc/secrets.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,15 @@ data "aws_secretsmanager_secret_version" "pagerduty_integration_keys" {
secret_id = data.aws_secretsmanager_secret.pagerduty_integration_keys.id
}

# Get the ARN of the logging bucket in `core-logging`
data "aws_secretsmanager_secret" "core_logging_bucket_arn" {
# Get the ARNs of the logging buckets in `core-logging`
data "aws_secretsmanager_secret" "core_logging_bucket_arns" {
provider = aws.modernisation-platform
name = "core_logging_bucket_arn"
name = "core_logging_bucket_arns"
}

data "aws_secretsmanager_secret_version" "core_logging_bucket_arn" {
data "aws_secretsmanager_secret_version" "core_logging_bucket_arns" {
provider = aws.modernisation-platform
secret_id = data.aws_secretsmanager_secret.core_logging_bucket_arn.id
secret_id = data.aws_secretsmanager_secret.core_logging_bucket_arns.id
}

# Data for Firehose Endpoint URL & Key that are held in secrets manager.
Expand Down