Skip to content

Commit

Permalink
Merge pull request #7729 from ministryofjustice/DSO/DSOS-2903/pagerdu…
Browse files Browse the repository at this point in the history 
…ty-alarm-improvement

Dso/dsos 2903/pagerduty alarm improvement
  • Loading branch information
@robertsweetman
robertsweetman authored Aug 20, 2024
2 parents 4367bd0 + f69e6eb commit 54ca28a
Show file tree
Hide file tree
Showing 2 changed files with 122 additions and 59 deletions.
120 changes: 61 additions & 59 deletions terraform/pagerduty/aws.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,65 +18,67 @@ resource "aws_secretsmanager_secret" "pagerduty_integration_keys" {
resource "aws_secretsmanager_secret_version" "pagerduty_integration_keys" {
secret_id = aws_secretsmanager_secret.pagerduty_integration_keys.id
secret_string = jsonencode({
core_alerts_cloudwatch = pagerduty_service_integration.core_alerts_cloudwatch.integration_key,
ddos_cloudwatch = pagerduty_service_integration.ddos_cloudwatch.integration_key,
tgw_cloudwatch = pagerduty_service_integration.tgw_cloudwatch.integration_key,
networking_cloudwatch = pagerduty_service_integration.networking_cloudwatch.integration_key,
operations_cloudwatch = pagerduty_service_integration.operations_cloudwatch.integration_key,
security_cloudwatch = pagerduty_service_integration.security_cloudwatch.integration_key,
nomis_alarms = pagerduty_service_integration.nomis_cloudwatch.integration_key,
nomis_nonprod_alarms = pagerduty_service_integration.nomis_nonprod_cloudwatch.integration_key,
laa_mlra_nonprod_alarms = pagerduty_service_integration.laa_mlra_nonprod_cloudwatch.integration_key,
laa_mlra_prod_alarms = pagerduty_service_integration.laa_mlra_prod_cloudwatch.integration_key,
laa_oas_nonprod_alarms = pagerduty_service_integration.laa_oas_nonprod_cloudwatch.integration_key,
laa_oas_prod_alarms = pagerduty_service_integration.laa_oas_prod_cloudwatch.integration_key,
jitbit_nonprod_alarms = pagerduty_service_integration.jitbit_nonprod_cloudwatch.integration_key,
jitbit_prod_alarms = pagerduty_service_integration.jitbit_prod_cloudwatch.integration_key,
iaps_nonprod_alarms = pagerduty_service_integration.iaps_nonprod_cloudwatch.integration_key,
iaps_prod_alarms = pagerduty_service_integration.iaps_prod_cloudwatch.integration_key,
laa_mojfin_prod_alarms = pagerduty_service_integration.laa_mojfin_prod_cloudwatch.integration_key,
laa_mojfin_non_prod_alarms = pagerduty_service_integration.laa_mojfin_non_prod_cloudwatch.integration_key,
hmpps_shef_dba_high_priority = pagerduty_service_integration.hmpps_shef_dba_high_priority.integration_key,
hmpps_shef_dba_low_priority = pagerduty_service_integration.hmpps_shef_dba_low_priority.integration_key,
hmpps_shef_dba_non_prod = pagerduty_service_integration.hmpps_shef_dba_non_prod.integration_key,
oasys_alarms = pagerduty_service_integration.oasys_cloudwatch.integration_key,
oasys_nonprod_alarms = pagerduty_service_integration.oasys_nonprod_cloudwatch.integration_key
test_alarms = pagerduty_service_integration.test_alarms.integration_key,
laa_portal_nonprod_alarms = pagerduty_service_integration.laa_portal_nonprod_cloudwatch.integration_key,
laa_portal_prod_alarms = pagerduty_service_integration.laa_portal_prod_cloudwatch.integration_key
laa_maat_nonprod_alarms = pagerduty_service_integration.laa_maat_nonprod_cloudwatch.integration_key,
laa_maat_prod_alarm = pagerduty_service_integration.laa_maat_prod_cloudwatch.integration_key,
csr_alarms = pagerduty_service_integration.csr_cloudwatch.integration_key,
dpr_nonprod_alarms = pagerduty_service_integration.dpr_nonprod_cloudwatch.integration_key,
planetfm_alarms = pagerduty_service_integration.planetfm_cloudwatch.integration_key,
ncas_non_prod_alarms = pagerduty_service_integration.ncas_non_prod_cloudwatch.integration_key,
ncas_prod_alarms = pagerduty_service_integration.ncas_prod_cloudwatch.integration_key,
wardship_non_prod_alarms = pagerduty_service_integration.wardship_non_prod_cloudwatch.integration_key,
wardship_prod_alarms = pagerduty_service_integration.wardship_prod_cloudwatch.integration_key,
pra_non_prod_alarms = pagerduty_service_integration.pra_non_prod_cloudwatch.integration_key,
pra_prod_alarms = pagerduty_service_integration.pra_prod_cloudwatch.integration_key,
tipstaff_non_prod_alarms = pagerduty_service_integration.tipstaff_non_prod_cloudwatch.integration_key,
tipstaff_prod_alarms = pagerduty_service_integration.tipstaff_prod_cloudwatch.integration_key,
dacp_non_prod_alarms = pagerduty_service_integration.dacp_non_prod_cloudwatch.integration_key,
dacp_prod_alarms = pagerduty_service_integration.dacp_prod_cloudwatch.integration_key,
laa_maat_api_nonprod_alarms = pagerduty_service_integration.laa_maat_api_nonprod_cloudwatch.integration_key,
laa_maat_api_prod_alarms = pagerduty_service_integration.laa_maat_api_prod_cloudwatch.integration_key,
hmpps_domain_services_prod_alarms = pagerduty_service_integration.hmpps_domain_services_prod_cloudwatch.integration_key,
hmpps_domain_services_nonprod_alarms = pagerduty_service_integration.hmpps_domain_services_nonprod_cloudwatch.integration_key
delius_core_nonprod_alarms = pagerduty_service_integration.delius_core_nonprod_cloudwatch.integration_key
delius_nextcloud_nonprod_alarms = pagerduty_service_integration.delius_nextcloud_nonprod_cloudwatch.integration_key
delius_nextcloud_prod_alarms = pagerduty_service_integration.delius_nextcloud_prod_cloudwatch.integration_key
laa_cwa_nonprod_alarms = pagerduty_service_integration.cwa_non_prod.integration_key
laa_cwa_prod_alarms = pagerduty_service_integration.cwa_prod.integration_key
nomis_data_hub_nonprod_alarms = pagerduty_service_integration.ndh_non_prod.integration_key
nomis_data_hub_prod_alarms = pagerduty_service_integration.ndh_prod.integration_key
laa_apex_nonprod_alarms = pagerduty_service_integration.apex_non_prod.integration_key
laa_apex_prod_alarms = pagerduty_service_integration.apex_prod.integration_key
delius_mis_nonprod_alarms = pagerduty_service_integration.delius_mis_non_prod.integration_key
delius_mis_prod_alarms = pagerduty_service_integration.delius_mis_prod.integration_key
laa_edw_nonprod_alarms = pagerduty_service_integration.edw_non_prod.integration_key
laa_edw_prod_alarms = pagerduty_service_integration.edw_prod.integration_key
cdpt-ifs-alarms = pagerduty_service_integration.cdpt_ifs_cloudwatch.integration_key
core_alerts_cloudwatch = pagerduty_service_integration.core_alerts_cloudwatch.integration_key,
ddos_cloudwatch = pagerduty_service_integration.ddos_cloudwatch.integration_key,
tgw_cloudwatch = pagerduty_service_integration.tgw_cloudwatch.integration_key,
networking_cloudwatch = pagerduty_service_integration.networking_cloudwatch.integration_key,
operations_cloudwatch = pagerduty_service_integration.operations_cloudwatch.integration_key,
security_cloudwatch = pagerduty_service_integration.security_cloudwatch.integration_key,
nomis_alarms = pagerduty_service_integration.nomis_cloudwatch.integration_key,
nomis_nonprod_alarms = pagerduty_service_integration.nomis_nonprod_cloudwatch.integration_key,
laa_mlra_nonprod_alarms = pagerduty_service_integration.laa_mlra_nonprod_cloudwatch.integration_key,
laa_mlra_prod_alarms = pagerduty_service_integration.laa_mlra_prod_cloudwatch.integration_key,
laa_oas_nonprod_alarms = pagerduty_service_integration.laa_oas_nonprod_cloudwatch.integration_key,
laa_oas_prod_alarms = pagerduty_service_integration.laa_oas_prod_cloudwatch.integration_key,
jitbit_nonprod_alarms = pagerduty_service_integration.jitbit_nonprod_cloudwatch.integration_key,
jitbit_prod_alarms = pagerduty_service_integration.jitbit_prod_cloudwatch.integration_key,
iaps_nonprod_alarms = pagerduty_service_integration.iaps_nonprod_cloudwatch.integration_key,
iaps_prod_alarms = pagerduty_service_integration.iaps_prod_cloudwatch.integration_key,
laa_mojfin_prod_alarms = pagerduty_service_integration.laa_mojfin_prod_cloudwatch.integration_key,
laa_mojfin_non_prod_alarms = pagerduty_service_integration.laa_mojfin_non_prod_cloudwatch.integration_key,
hmpps_shef_dba_high_priority = pagerduty_service_integration.hmpps_shef_dba_high_priority.integration_key,
hmpps_shef_dba_low_priority = pagerduty_service_integration.hmpps_shef_dba_low_priority.integration_key,
hmpps_shef_dba_non_prod = pagerduty_service_integration.hmpps_shef_dba_non_prod.integration_key,
oasys_alarms = pagerduty_service_integration.oasys_cloudwatch.integration_key,
oasys_nonprod_alarms = pagerduty_service_integration.oasys_nonprod_cloudwatch.integration_key
test_alarms = pagerduty_service_integration.test_alarms.integration_key,
laa_portal_nonprod_alarms = pagerduty_service_integration.laa_portal_nonprod_cloudwatch.integration_key,
laa_portal_prod_alarms = pagerduty_service_integration.laa_portal_prod_cloudwatch.integration_key
laa_maat_nonprod_alarms = pagerduty_service_integration.laa_maat_nonprod_cloudwatch.integration_key,
laa_maat_prod_alarm = pagerduty_service_integration.laa_maat_prod_cloudwatch.integration_key,
csr_alarms = pagerduty_service_integration.csr_cloudwatch.integration_key,
dpr_nonprod_alarms = pagerduty_service_integration.dpr_nonprod_cloudwatch.integration_key,
planetfm_alarms = pagerduty_service_integration.planetfm_cloudwatch.integration_key,
ncas_non_prod_alarms = pagerduty_service_integration.ncas_non_prod_cloudwatch.integration_key,
ncas_prod_alarms = pagerduty_service_integration.ncas_prod_cloudwatch.integration_key,
wardship_non_prod_alarms = pagerduty_service_integration.wardship_non_prod_cloudwatch.integration_key,
wardship_prod_alarms = pagerduty_service_integration.wardship_prod_cloudwatch.integration_key,
pra_non_prod_alarms = pagerduty_service_integration.pra_non_prod_cloudwatch.integration_key,
pra_prod_alarms = pagerduty_service_integration.pra_prod_cloudwatch.integration_key,
tipstaff_non_prod_alarms = pagerduty_service_integration.tipstaff_non_prod_cloudwatch.integration_key,
tipstaff_prod_alarms = pagerduty_service_integration.tipstaff_prod_cloudwatch.integration_key,
dacp_non_prod_alarms = pagerduty_service_integration.dacp_non_prod_cloudwatch.integration_key,
dacp_prod_alarms = pagerduty_service_integration.dacp_prod_cloudwatch.integration_key,
laa_maat_api_nonprod_alarms = pagerduty_service_integration.laa_maat_api_nonprod_cloudwatch.integration_key,
laa_maat_api_prod_alarms = pagerduty_service_integration.laa_maat_api_prod_cloudwatch.integration_key,
hmpps_domain_services_prod_alarms = pagerduty_service_integration.hmpps_domain_services_prod_cloudwatch.integration_key,
hmpps_domain_services_nonprod_alarms = pagerduty_service_integration.hmpps_domain_services_nonprod_cloudwatch.integration_key
delius_core_nonprod_alarms = pagerduty_service_integration.delius_core_nonprod_cloudwatch.integration_key
delius_nextcloud_nonprod_alarms = pagerduty_service_integration.delius_nextcloud_nonprod_cloudwatch.integration_key
delius_nextcloud_prod_alarms = pagerduty_service_integration.delius_nextcloud_prod_cloudwatch.integration_key
laa_cwa_nonprod_alarms = pagerduty_service_integration.cwa_non_prod.integration_key
laa_cwa_prod_alarms = pagerduty_service_integration.cwa_prod.integration_key
nomis_data_hub_nonprod_alarms = pagerduty_service_integration.ndh_non_prod.integration_key
nomis_data_hub_prod_alarms = pagerduty_service_integration.ndh_prod.integration_key
laa_apex_nonprod_alarms = pagerduty_service_integration.apex_non_prod.integration_key
laa_apex_prod_alarms = pagerduty_service_integration.apex_prod.integration_key
delius_mis_nonprod_alarms = pagerduty_service_integration.delius_mis_non_prod.integration_key
delius_mis_prod_alarms = pagerduty_service_integration.delius_mis_prod.integration_key
laa_edw_nonprod_alarms = pagerduty_service_integration.edw_non_prod.integration_key
laa_edw_prod_alarms = pagerduty_service_integration.edw_prod.integration_key
cdpt-ifs-alarms = pagerduty_service_integration.cdpt_ifs_cloudwatch.integration_key
corporate-staff-rostering-preproduction = pagerduty_service_integration.integrations["corporate-staff-rostering-preproduction"].integration_key
corporate-staff-rostering-production = pagerduty_service_integration.integrations["corporate-staff-rostering-production"].integration_key
})
}

Expand Down
61 changes: 61 additions & 0 deletions terraform/pagerduty/member-services-integrations.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2543,3 +2543,64 @@ resource "pagerduty_slack_connection" "chaps_slack" {
priorities = ["*"]
}
}

locals {
services = {
corporate-staff-rostering-preproduction = { slack_channel_id = "C07J1UFEK25" } # corporate-staff-rostering-alarms-non-prod
corporate-staff-rostering-production = { slack_channel_id = "C07HQ17MY11" } # corporate-staff-rostering-alarms-prod
# nomis_development = { slack_channel_id = "TBD" } # nomis-alarms-non-prod
# nomis_test = { slack_channel_id = "TBD" } # nomis-alarms-non-prod
# nomis_preproduction = { slack_channel_id = "TBD" } # nomis-alarms-non-prod
# nomis_production = { slack_channel_id = "TBD" } # nomis-alarms-prod
# planetfm_preproduction = { slack_channel_id = "TBD" } # planetfm-alarms-non-prod
# planetfm_production = { slack_channel_id = "TBD" } # planetfm-alarms-prod
}
slack_events = [
"incident.triggered",
"incident.acknowledged",
"incident.escalated",
"incident.resolved",
"incident.reassigned",
"incident.annotated",
"incident.unacknowledged",
"incident.delegated",
"incident.priority_updated",
"incident.responder.added",
"incident.responder.replied",
"incident.action_invocation.created",
"incident.action_invocation.terminated",
"incident.action_invocation.updated",
"incident.status_update_published",
"incident.reopened"
]
}

resource "pagerduty_service" "services" {
for_each = local.services

name = each.key
description = "${each.key}_alarms"
auto_resolve_timeout = 345600
acknowledgement_timeout = "null"
escalation_policy = pagerduty_escalation_policy.member_policy.id
alert_creation = "create_alerts_and_incidents"
}
resource "pagerduty_service_integration" "integrations" {
for_each = pagerduty_service.services
name = data.pagerduty_vendor.cloudwatch.name
service = each.value.id
vendor = data.pagerduty_vendor.cloudwatch.id
}

resource "pagerduty_slack_connection" "connections" {
for_each = local.services
source_id = pagerduty_service.services[each.key].id
source_type = "service_reference"
workspace_id = local.slack_workspace_id
channel_id = each.value.slack_channel_id
notification_type = "responder"
config {
events = local.slack_events
priorities = ["*"]
}
}

0 comments on commit 54ca28a

Please sign in to comment.