Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dso/dsos 2903/pagerduty alarm improvement #7729

Merged
merged 9 commits into from
Aug 20, 2024
Merged

Dso/dsos 2903/pagerduty alarm improvement #7729

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
5d48ff7
example for pagerduty integrations
robertsweetman Aug 20, 2024
2eb8b0b
change for_each loop
robertsweetman Aug 20, 2024
b86b18d
creating new integrations for new alarms
robertsweetman Aug 20, 2024
6dc9e0c
change root module declaration
robertsweetman Aug 20, 2024
40b0e0f
change service name
robertsweetman Aug 20, 2024
ae2ae23
fix dependency issue
robertsweetman Aug 20, 2024
07e3c28
refer to missing value correctly
robertsweetman Aug 20, 2024
ee663b6
change the nonprod channel name again
robertsweetman Aug 20, 2024
f69e6eb
use full application naming
robertsweetman Aug 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 61 additions & 59 deletions terraform/pagerduty/aws.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,65 +18,67 @@ resource "aws_secretsmanager_secret" "pagerduty_integration_keys" {
resource "aws_secretsmanager_secret_version" "pagerduty_integration_keys" {
secret_id = aws_secretsmanager_secret.pagerduty_integration_keys.id
secret_string = jsonencode({
core_alerts_cloudwatch = pagerduty_service_integration.core_alerts_cloudwatch.integration_key,
ddos_cloudwatch = pagerduty_service_integration.ddos_cloudwatch.integration_key,
tgw_cloudwatch = pagerduty_service_integration.tgw_cloudwatch.integration_key,
networking_cloudwatch = pagerduty_service_integration.networking_cloudwatch.integration_key,
operations_cloudwatch = pagerduty_service_integration.operations_cloudwatch.integration_key,
security_cloudwatch = pagerduty_service_integration.security_cloudwatch.integration_key,
nomis_alarms = pagerduty_service_integration.nomis_cloudwatch.integration_key,
nomis_nonprod_alarms = pagerduty_service_integration.nomis_nonprod_cloudwatch.integration_key,
laa_mlra_nonprod_alarms = pagerduty_service_integration.laa_mlra_nonprod_cloudwatch.integration_key,
laa_mlra_prod_alarms = pagerduty_service_integration.laa_mlra_prod_cloudwatch.integration_key,
laa_oas_nonprod_alarms = pagerduty_service_integration.laa_oas_nonprod_cloudwatch.integration_key,
laa_oas_prod_alarms = pagerduty_service_integration.laa_oas_prod_cloudwatch.integration_key,
jitbit_nonprod_alarms = pagerduty_service_integration.jitbit_nonprod_cloudwatch.integration_key,
jitbit_prod_alarms = pagerduty_service_integration.jitbit_prod_cloudwatch.integration_key,
iaps_nonprod_alarms = pagerduty_service_integration.iaps_nonprod_cloudwatch.integration_key,
iaps_prod_alarms = pagerduty_service_integration.iaps_prod_cloudwatch.integration_key,
laa_mojfin_prod_alarms = pagerduty_service_integration.laa_mojfin_prod_cloudwatch.integration_key,
laa_mojfin_non_prod_alarms = pagerduty_service_integration.laa_mojfin_non_prod_cloudwatch.integration_key,
hmpps_shef_dba_high_priority = pagerduty_service_integration.hmpps_shef_dba_high_priority.integration_key,
hmpps_shef_dba_low_priority = pagerduty_service_integration.hmpps_shef_dba_low_priority.integration_key,
hmpps_shef_dba_non_prod = pagerduty_service_integration.hmpps_shef_dba_non_prod.integration_key,
oasys_alarms = pagerduty_service_integration.oasys_cloudwatch.integration_key,
oasys_nonprod_alarms = pagerduty_service_integration.oasys_nonprod_cloudwatch.integration_key
test_alarms = pagerduty_service_integration.test_alarms.integration_key,
laa_portal_nonprod_alarms = pagerduty_service_integration.laa_portal_nonprod_cloudwatch.integration_key,
laa_portal_prod_alarms = pagerduty_service_integration.laa_portal_prod_cloudwatch.integration_key
laa_maat_nonprod_alarms = pagerduty_service_integration.laa_maat_nonprod_cloudwatch.integration_key,
laa_maat_prod_alarm = pagerduty_service_integration.laa_maat_prod_cloudwatch.integration_key,
csr_alarms = pagerduty_service_integration.csr_cloudwatch.integration_key,
dpr_nonprod_alarms = pagerduty_service_integration.dpr_nonprod_cloudwatch.integration_key,
planetfm_alarms = pagerduty_service_integration.planetfm_cloudwatch.integration_key,
ncas_non_prod_alarms = pagerduty_service_integration.ncas_non_prod_cloudwatch.integration_key,
ncas_prod_alarms = pagerduty_service_integration.ncas_prod_cloudwatch.integration_key,
wardship_non_prod_alarms = pagerduty_service_integration.wardship_non_prod_cloudwatch.integration_key,
wardship_prod_alarms = pagerduty_service_integration.wardship_prod_cloudwatch.integration_key,
pra_non_prod_alarms = pagerduty_service_integration.pra_non_prod_cloudwatch.integration_key,
pra_prod_alarms = pagerduty_service_integration.pra_prod_cloudwatch.integration_key,
tipstaff_non_prod_alarms = pagerduty_service_integration.tipstaff_non_prod_cloudwatch.integration_key,
tipstaff_prod_alarms = pagerduty_service_integration.tipstaff_prod_cloudwatch.integration_key,
dacp_non_prod_alarms = pagerduty_service_integration.dacp_non_prod_cloudwatch.integration_key,
dacp_prod_alarms = pagerduty_service_integration.dacp_prod_cloudwatch.integration_key,
laa_maat_api_nonprod_alarms = pagerduty_service_integration.laa_maat_api_nonprod_cloudwatch.integration_key,
laa_maat_api_prod_alarms = pagerduty_service_integration.laa_maat_api_prod_cloudwatch.integration_key,
hmpps_domain_services_prod_alarms = pagerduty_service_integration.hmpps_domain_services_prod_cloudwatch.integration_key,
hmpps_domain_services_nonprod_alarms = pagerduty_service_integration.hmpps_domain_services_nonprod_cloudwatch.integration_key
delius_core_nonprod_alarms = pagerduty_service_integration.delius_core_nonprod_cloudwatch.integration_key
delius_nextcloud_nonprod_alarms = pagerduty_service_integration.delius_nextcloud_nonprod_cloudwatch.integration_key
delius_nextcloud_prod_alarms = pagerduty_service_integration.delius_nextcloud_prod_cloudwatch.integration_key
laa_cwa_nonprod_alarms = pagerduty_service_integration.cwa_non_prod.integration_key
laa_cwa_prod_alarms = pagerduty_service_integration.cwa_prod.integration_key
nomis_data_hub_nonprod_alarms = pagerduty_service_integration.ndh_non_prod.integration_key
nomis_data_hub_prod_alarms = pagerduty_service_integration.ndh_prod.integration_key
laa_apex_nonprod_alarms = pagerduty_service_integration.apex_non_prod.integration_key
laa_apex_prod_alarms = pagerduty_service_integration.apex_prod.integration_key
delius_mis_nonprod_alarms = pagerduty_service_integration.delius_mis_non_prod.integration_key
delius_mis_prod_alarms = pagerduty_service_integration.delius_mis_prod.integration_key
laa_edw_nonprod_alarms = pagerduty_service_integration.edw_non_prod.integration_key
laa_edw_prod_alarms = pagerduty_service_integration.edw_prod.integration_key
cdpt-ifs-alarms = pagerduty_service_integration.cdpt_ifs_cloudwatch.integration_key
core_alerts_cloudwatch = pagerduty_service_integration.core_alerts_cloudwatch.integration_key,
ddos_cloudwatch = pagerduty_service_integration.ddos_cloudwatch.integration_key,
tgw_cloudwatch = pagerduty_service_integration.tgw_cloudwatch.integration_key,
networking_cloudwatch = pagerduty_service_integration.networking_cloudwatch.integration_key,
operations_cloudwatch = pagerduty_service_integration.operations_cloudwatch.integration_key,
security_cloudwatch = pagerduty_service_integration.security_cloudwatch.integration_key,
nomis_alarms = pagerduty_service_integration.nomis_cloudwatch.integration_key,
nomis_nonprod_alarms = pagerduty_service_integration.nomis_nonprod_cloudwatch.integration_key,
laa_mlra_nonprod_alarms = pagerduty_service_integration.laa_mlra_nonprod_cloudwatch.integration_key,
laa_mlra_prod_alarms = pagerduty_service_integration.laa_mlra_prod_cloudwatch.integration_key,
laa_oas_nonprod_alarms = pagerduty_service_integration.laa_oas_nonprod_cloudwatch.integration_key,
laa_oas_prod_alarms = pagerduty_service_integration.laa_oas_prod_cloudwatch.integration_key,
jitbit_nonprod_alarms = pagerduty_service_integration.jitbit_nonprod_cloudwatch.integration_key,
jitbit_prod_alarms = pagerduty_service_integration.jitbit_prod_cloudwatch.integration_key,
iaps_nonprod_alarms = pagerduty_service_integration.iaps_nonprod_cloudwatch.integration_key,
iaps_prod_alarms = pagerduty_service_integration.iaps_prod_cloudwatch.integration_key,
laa_mojfin_prod_alarms = pagerduty_service_integration.laa_mojfin_prod_cloudwatch.integration_key,
laa_mojfin_non_prod_alarms = pagerduty_service_integration.laa_mojfin_non_prod_cloudwatch.integration_key,
hmpps_shef_dba_high_priority = pagerduty_service_integration.hmpps_shef_dba_high_priority.integration_key,
hmpps_shef_dba_low_priority = pagerduty_service_integration.hmpps_shef_dba_low_priority.integration_key,
hmpps_shef_dba_non_prod = pagerduty_service_integration.hmpps_shef_dba_non_prod.integration_key,
oasys_alarms = pagerduty_service_integration.oasys_cloudwatch.integration_key,
oasys_nonprod_alarms = pagerduty_service_integration.oasys_nonprod_cloudwatch.integration_key
test_alarms = pagerduty_service_integration.test_alarms.integration_key,
laa_portal_nonprod_alarms = pagerduty_service_integration.laa_portal_nonprod_cloudwatch.integration_key,
laa_portal_prod_alarms = pagerduty_service_integration.laa_portal_prod_cloudwatch.integration_key
laa_maat_nonprod_alarms = pagerduty_service_integration.laa_maat_nonprod_cloudwatch.integration_key,
laa_maat_prod_alarm = pagerduty_service_integration.laa_maat_prod_cloudwatch.integration_key,
csr_alarms = pagerduty_service_integration.csr_cloudwatch.integration_key,
dpr_nonprod_alarms = pagerduty_service_integration.dpr_nonprod_cloudwatch.integration_key,
planetfm_alarms = pagerduty_service_integration.planetfm_cloudwatch.integration_key,
ncas_non_prod_alarms = pagerduty_service_integration.ncas_non_prod_cloudwatch.integration_key,
ncas_prod_alarms = pagerduty_service_integration.ncas_prod_cloudwatch.integration_key,
wardship_non_prod_alarms = pagerduty_service_integration.wardship_non_prod_cloudwatch.integration_key,
wardship_prod_alarms = pagerduty_service_integration.wardship_prod_cloudwatch.integration_key,
pra_non_prod_alarms = pagerduty_service_integration.pra_non_prod_cloudwatch.integration_key,
pra_prod_alarms = pagerduty_service_integration.pra_prod_cloudwatch.integration_key,
tipstaff_non_prod_alarms = pagerduty_service_integration.tipstaff_non_prod_cloudwatch.integration_key,
tipstaff_prod_alarms = pagerduty_service_integration.tipstaff_prod_cloudwatch.integration_key,
dacp_non_prod_alarms = pagerduty_service_integration.dacp_non_prod_cloudwatch.integration_key,
dacp_prod_alarms = pagerduty_service_integration.dacp_prod_cloudwatch.integration_key,
laa_maat_api_nonprod_alarms = pagerduty_service_integration.laa_maat_api_nonprod_cloudwatch.integration_key,
laa_maat_api_prod_alarms = pagerduty_service_integration.laa_maat_api_prod_cloudwatch.integration_key,
hmpps_domain_services_prod_alarms = pagerduty_service_integration.hmpps_domain_services_prod_cloudwatch.integration_key,
hmpps_domain_services_nonprod_alarms = pagerduty_service_integration.hmpps_domain_services_nonprod_cloudwatch.integration_key
delius_core_nonprod_alarms = pagerduty_service_integration.delius_core_nonprod_cloudwatch.integration_key
delius_nextcloud_nonprod_alarms = pagerduty_service_integration.delius_nextcloud_nonprod_cloudwatch.integration_key
delius_nextcloud_prod_alarms = pagerduty_service_integration.delius_nextcloud_prod_cloudwatch.integration_key
laa_cwa_nonprod_alarms = pagerduty_service_integration.cwa_non_prod.integration_key
laa_cwa_prod_alarms = pagerduty_service_integration.cwa_prod.integration_key
nomis_data_hub_nonprod_alarms = pagerduty_service_integration.ndh_non_prod.integration_key
nomis_data_hub_prod_alarms = pagerduty_service_integration.ndh_prod.integration_key
laa_apex_nonprod_alarms = pagerduty_service_integration.apex_non_prod.integration_key
laa_apex_prod_alarms = pagerduty_service_integration.apex_prod.integration_key
delius_mis_nonprod_alarms = pagerduty_service_integration.delius_mis_non_prod.integration_key
delius_mis_prod_alarms = pagerduty_service_integration.delius_mis_prod.integration_key
laa_edw_nonprod_alarms = pagerduty_service_integration.edw_non_prod.integration_key
laa_edw_prod_alarms = pagerduty_service_integration.edw_prod.integration_key
cdpt-ifs-alarms = pagerduty_service_integration.cdpt_ifs_cloudwatch.integration_key
corporate-staff-rostering-preproduction = pagerduty_service_integration.integrations["corporate-staff-rostering-preproduction"].integration_key
corporate-staff-rostering-production = pagerduty_service_integration.integrations["corporate-staff-rostering-production"].integration_key
})
}

Expand Down
61 changes: 61 additions & 0 deletions terraform/pagerduty/member-services-integrations.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2543,3 +2543,64 @@ resource "pagerduty_slack_connection" "chaps_slack" {
priorities = ["*"]
}
}

locals {
services = {
corporate-staff-rostering-preproduction = { slack_channel_id = "C07J1UFEK25" } # corporate-staff-rostering-alarms-non-prod
corporate-staff-rostering-production = { slack_channel_id = "C07HQ17MY11" } # corporate-staff-rostering-alarms-prod
# nomis_development = { slack_channel_id = "TBD" } # nomis-alarms-non-prod
# nomis_test = { slack_channel_id = "TBD" } # nomis-alarms-non-prod
# nomis_preproduction = { slack_channel_id = "TBD" } # nomis-alarms-non-prod
# nomis_production = { slack_channel_id = "TBD" } # nomis-alarms-prod
# planetfm_preproduction = { slack_channel_id = "TBD" } # planetfm-alarms-non-prod
# planetfm_production = { slack_channel_id = "TBD" } # planetfm-alarms-prod
}
slack_events = [
"incident.triggered",
"incident.acknowledged",
"incident.escalated",
"incident.resolved",
"incident.reassigned",
"incident.annotated",
"incident.unacknowledged",
"incident.delegated",
"incident.priority_updated",
"incident.responder.added",
"incident.responder.replied",
"incident.action_invocation.created",
"incident.action_invocation.terminated",
"incident.action_invocation.updated",
"incident.status_update_published",
"incident.reopened"
]
}

resource "pagerduty_service" "services" {
for_each = local.services

name = each.key
description = "${each.key}_alarms"
auto_resolve_timeout = 345600
acknowledgement_timeout = "null"
escalation_policy = pagerduty_escalation_policy.member_policy.id
alert_creation = "create_alerts_and_incidents"
}
resource "pagerduty_service_integration" "integrations" {
for_each = pagerduty_service.services
name = data.pagerduty_vendor.cloudwatch.name
service = each.value.id
vendor = data.pagerduty_vendor.cloudwatch.id
}

resource "pagerduty_slack_connection" "connections" {
for_each = local.services
source_id = pagerduty_service.services[each.key].id
source_type = "service_reference"
workspace_id = local.slack_workspace_id
channel_id = each.value.slack_channel_id
notification_type = "responder"
config {
events = local.slack_events
priorities = ["*"]
}
}