Merge pull request #14 from wellcomecollection/max-age-alarm

Add queue alarm on message age

RELEASE.md

@@ -0,0 +1,15 @@
+RELEASE_TYPE: minor
+
+This change adds a main queue alarm that triggers when the main queue has messages older than a certain age. 
+
+This is useful for monitoring the health of the queue and ensuring that messages are being processed in a timely manner.
+
+Adds the variables:
+
+- `dlq_alarm_action_arns` - The ARNs of the resources to send DLQ alarm notifications to
+- `main_q_age_alarm_action_arns` - The ARN of the resources to send main queue age alarm notifications to
+- `max_age_in_hours` - The maximum age of a message in the main queue before the alarm triggers
+- `enable_dlq_not_empty_alarm` - Whether to enable the DLQ not empty alarm (default: `false`), overridden if `dlq_alarm_action_arns` is not empty
+- `enable_main_q_age_alarm` - Whether to enable the main queue age alarm (default: `false`), overridden if `main_q_age_alarm_action_arns` is not empty
+
+We deprecate the `alarm_topic_arn` variable in favour of the new `dlq_alarm_action_arns` and `main_q_age_alarm_action_arns` variables.

queue/alarms.tf

@@ -1,5 +1,16 @@
+locals {
+  max_age_in_seconds = var.max_age_in_hours * 3600
+
+  # Allows for deprecation of alarm_topic_arn in favor of dlq_alarm_topic_arn
+  alarm_topic_arn_safe  = var.alarm_topic_arn != null ? [var.alarm_topic_arn] : []
+  dlq_alarm_action_arns = var.dlq_alarm_action_arns != [] ? var.dlq_alarm_action_arns : local.alarm_topic_arn_safe
+
+  enable_dlq_not_empty_alarm = var.enable_dlq_not_empty_alarm || local.dlq_alarm_action_arns != []
+  enable_queue_age_alarm     = var.enable_queue_age_alarm || var.main_q_age_alarm_action_arns != []
+}
+
 resource "aws_cloudwatch_metric_alarm" "dlq_not_empty" {
-  count = var.alarm_topic_arn != null ? 1 : 0
+  count = local.enable_dlq_not_empty_alarm == true ? 1 : 0
 
   alarm_name          = "${aws_sqs_queue.dlq.name}_not_empty"
   comparison_operator = "GreaterThanThreshold"
@@ -10,10 +21,32 @@ resource "aws_cloudwatch_metric_alarm" "dlq_not_empty" {
   threshold           = 0
   statistic           = "Average"
 
+  alarm_description = "Alarm if the DLQ is not empty"
+
   dimensions = {
     QueueName = aws_sqs_queue.dlq.name
   }
 
-  alarm_actions = [var.alarm_topic_arn]
+  alarm_actions = local.dlq_alarm_action_arns
 }
 
+resource "aws_cloudwatch_metric_alarm" "queue_age" {
+  count = local.enable_queue_age_alarm == true ? 1 : 0
+
+  alarm_name          = "${aws_sqs_queue.q.name}_age"
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = 1
+  metric_name         = "ApproximateAgeOfOldestMessage"
+  namespace           = "AWS/SQS"
+  period              = 60
+  threshold           = local.max_age_in_seconds
+  statistic           = "Maximum"
+
+  alarm_description = "Message age exceeds ${var.max_age_in_hours} hours"
+
+  dimensions = {
+    QueueName = aws_sqs_queue.q.name
+  }
+
+  alarm_actions = var.main_q_age_alarm_action_arns
+}

queue/variables.tf

@@ -38,10 +38,36 @@ variable "max_receive_count" {
 }
 
 variable "alarm_topic_arn" {
-  description = "ARN of the topic where to send notification for DLQs not being empty. If null, no alarm will be created."
+  description = "DEPRECATED, use dlq_alarm_topic_arn: ARN of the topic where to send notification for DLQs not being empty. If null, no alarm will be created."
   default     = null
 }
 
+variable "enable_dlq_not_empty_alarm" {
+  description = "Enable alarm for DLQs not being empty"
+  default     = false
+}
+
+variable "enable_queue_age_alarm" {
+  description = "Enable alarm for messages exceeding max_age_in_hours"
+  default     = false
+}
+
+variable "dlq_alarm_action_arns" {
+  description = "ARNs for the topics where to send notification for DLQs not being empty, if not empty overrides alarm_topic_arn."
+  default     = []
+}
+
+variable "main_q_age_alarm_action_arns" {
+  description = "ARN for the topics where to send notification for messages exceeding max_age_in_hours, if not empty overrides enable_queue_age_alarm."
+  default     = []
+}
+
+variable "max_age_in_hours" {
+  description = "The maximum age of a message in hours"
+  type        = number
+  default     = 6
+}
+
 variable "fifo_queue" {
   description = "Boolean designating a FIFO queue"
   default     = false