Added querier autoscaling panels and alerts to mixin (#1006)

* Added querier autoscaling panels and alerts to mixin Signed-off-by: Marco Pracucci <marco@pracucci.com> * Fixed typos Signed-off-by: Marco Pracucci <marco@pracucci.com> * Addressed review comments Signed-off-by: Marco Pracucci <marco@pracucci.com>
grafana · Feb 3, 2022 · 91b235e · 91b235e
1 parent f4a866e
commit 91b235e
Show file tree

Hide file tree

Showing 14 changed files with 510 additions and 43 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -468,6 +468,7 @@
 * [CHANGE] Enabled resources dashboards by default. Can be disabled setting `resources_dashboards_enabled` config field to `false`. #920
 * [FEATURE] Added `Cortex / Overrides` dashboard, displaying default limits and per-tenant overrides applied to Mimir. #673
 * [FEATURE] Added `Mimir / Tenants` and `Mimir / Top tenants` dashboards, displaying user-based metrics. #776
+* [FEATURE] Added querier autoscaling panels and alerts. #1006
 * [ENHANCEMENT] cortex-mixin: Make `cluster_namespace_deployment:kube_pod_container_resource_requests_{cpu_cores,memory_bytes}:sum` backwards compatible with `kube-state-metrics` v2.0.0. [#317](https://github.com/grafana/cortex-jsonnet/pull/317)
 * [ENHANCEMENT] Cortex-mixin: Include `cortex-gw-internal` naming variation in default `gateway` job names. [#328](https://github.com/grafana/cortex-jsonnet/pull/328)
 * [ENHANCEMENT] Ruler dashboard: added object storage metrics. [#354](https://github.com/grafana/cortex-jsonnet/pull/354)

diff --git a/Makefile b/Makefile
@@ -437,7 +437,7 @@ check-mixin-playbook: build-mixin
 
 build-mixin: check-mixin-jb
 	@rm -rf $(MIXIN_OUT_PATH) && mkdir $(MIXIN_OUT_PATH)
-	@mixtool generate all --output-alerts $(MIXIN_OUT_PATH)/alerts.yaml --output-rules $(MIXIN_OUT_PATH)/rules.yaml --directory $(MIXIN_OUT_PATH)/dashboards ${MIXIN_PATH}/mixin.libsonnet
+	@mixtool generate all --output-alerts $(MIXIN_OUT_PATH)/alerts.yaml --output-rules $(MIXIN_OUT_PATH)/rules.yaml --directory $(MIXIN_OUT_PATH)/dashboards ${MIXIN_PATH}/mixin-compiled.libsonnet
 	@cd $(MIXIN_OUT_PATH)/.. && zip -q -r mimir-mixin.zip $$(basename "$(MIXIN_OUT_PATH)")
 	@echo "The mixin has been compiled to $(MIXIN_OUT_PATH) and archived to $$(realpath --relative-to=$$(pwd) $(MIXIN_OUT_PATH)/../mimir-mixin.zip)"
 

diff --git a/operations/mimir-mixin-compiled/alerts.yaml b/operations/mimir-mixin-compiled/alerts.yaml
@@ -122,7 +122,7 @@ groups:
     for: 5m
     labels:
       severity: critical
-- name: cortex_instance_limits_alerts
+- name: mimir_instance_limits_alerts
   rules:
   - alert: MimirIngesterReachingSeriesLimit
     annotations:
@@ -436,7 +436,7 @@ groups:
       increase(cortex_alertmanager_state_initial_sync_completed_total{outcome="failed"}[1m]) > 0
     labels:
       severity: critical
-- name: cortex_blocks_alerts
+- name: mimir_blocks_alerts
   rules:
   - alert: MimirIngesterHasNotShippedBlocks
     annotations:
@@ -580,7 +580,7 @@ groups:
     for: 6h
     labels:
       severity: warning
-- name: cortex_compactor_alerts
+- name: mimir_compactor_alerts
   rules:
   - alert: MimirCompactorHasNotSuccessfullyCleanedUpBlocks
     annotations:
@@ -641,3 +641,15 @@ groups:
     for: 1m
     labels:
       severity: warning
+- name: mimir_autoscaling_querier
+  rules:
+  - alert: MimirQuerierAutoscalerNotActive
+    annotations:
+      message: The Horizontal Pod Autoscaler (HPA) in {{ $labels.namespace }} is not active.
+    expr: |
+      kube_horizontalpodautoscaler_status_condition{horizontalpodautoscaler="keda-hpa-querier",condition="ScalingActive",status="false"}
+      * on(cluster, namespace) group_left max by(cluster, namespace) (cortex_build_info)
+      > 0
+    for: 15m
+    labels:
+      severity: warning