diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fa0230d760..3da4987a7e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -254,6 +254,7 @@ * [ENHANCEMENT] Dashboards: Include block compaction level on queried blocks in 'Mimir / Queries' dashboard. #9706 * [ENHANCEMENT] Alerts: add `MimirIngesterMissedRecordsFromKafka` to detect gaps in consumed records in the ingester when using the experimental Kafka-based storage. #9921 #9972 * [ENHANCEMENT] Dashboards: Add more panels to 'Mimir / Writes' for concurrent ingestion and fetching when using ingest storage. #10021 +* [ENHANCEMENT] Dashboards: Include CPU and memory resources in 'Mimir / Ruler' dashboard. #10656 * [BUGFIX] Dashboards: Fix autoscaling metrics joins when series churn. #9412 #9450 #9432 * [BUGFIX] Alerts: Fix autoscaling metrics joins in `MimirAutoscalerNotActive` when series churn. #9412 * [BUGFIX] Alerts: Exclude failed cache "add" operations from alerting since failures are expected in normal operation. #9658 diff --git a/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml b/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml index 002a90d76ac..fefb7b3483f 100644 --- a/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml +++ b/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml @@ -31419,6 +31419,288 @@ data: "title": "Reads (ingesters)", "titleSize": "h6" }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash" + } + } + ] + } + ] + }, + "id": 11, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})", + "format": "time_series", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\",resource=\"cpu\"})", + "format": "time_series", + "legendFormat": "request", + "legendLink": null + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash" + } + } + ] + } + ] + }, + "id": 12, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"} > 0)", + "format": "time_series", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\",resource=\"memory\"})", + "format": "time_series", + "legendFormat": "request", + "legendLink": null + } + ], + "title": "Memory (workingset)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "id": 13, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + } + ], + "title": "Memory (go heap inuse)", + "type": "timeseries" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Ruler resources", + "titleSize": "h6" + }, { "collapse": false, "height": "250px", @@ -31584,7 +31866,7 @@ data: } ] }, - "id": 11, + "id": 14, "links": [ ], "options": { "legend": { @@ -31632,7 +31914,7 @@ data: }, "overrides": [ ] }, - "id": 12, + "id": 15, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -31723,7 +32005,7 @@ data: }, "overrides": [ ] }, - "id": 13, + "id": 16, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -31802,7 +32084,7 @@ data: }, "overrides": [ ] }, - "id": 14, + "id": 17, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -31899,7 +32181,7 @@ data: } ] }, - "id": 15, + "id": 18, "links": [ ], "options": { "legend": { @@ -31960,7 +32242,7 @@ data: }, "overrides": [ ] }, - "id": 16, + "id": 19, "links": [ ], "options": { "legend": { @@ -32009,7 +32291,7 @@ data: }, "overrides": [ ] }, - "id": 17, + "id": 20, "links": [ ], "options": { "legend": { @@ -32058,7 +32340,7 @@ data: }, "overrides": [ ] }, - "id": 18, + "id": 21, "links": [ ], "options": { "legend": { @@ -32118,7 +32400,7 @@ data: }, "overrides": [ ] }, - "id": 19, + "id": 22, "links": [ ], "options": { "legend": { @@ -32166,7 +32448,7 @@ data: }, "overrides": [ ] }, - "id": 20, + "id": 23, "links": [ ], "options": { "legend": { @@ -32214,7 +32496,7 @@ data: }, "overrides": [ ] }, - "id": 21, + "id": 24, "links": [ ], "options": { "legend": { @@ -32274,7 +32556,7 @@ data: }, "overrides": [ ] }, - "id": 22, + "id": 25, "links": [ ], "options": { "legend": { @@ -32334,7 +32616,7 @@ data: }, "overrides": [ ] }, - "id": 23, + "id": 26, "links": [ ], "options": { "legend": { @@ -32367,7 +32649,7 @@ data: "unit": "percentunit" } }, - "id": 24, + "id": 27, "links": [ ], "options": { "legend": { @@ -32415,7 +32697,7 @@ data: }, "overrides": [ ] }, - "id": 25, + "id": 28, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -32494,7 +32776,7 @@ data: }, "overrides": [ ] }, - "id": 26, + "id": 29, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -32585,7 +32867,7 @@ data: }, "overrides": [ ] }, - "id": 27, + "id": 30, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -32664,7 +32946,7 @@ data: }, "overrides": [ ] }, - "id": 28, + "id": 31, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -32743,7 +33025,7 @@ data: }, "overrides": [ ] }, - "id": 29, + "id": 32, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -32822,7 +33104,7 @@ data: }, "overrides": [ ] }, - "id": 30, + "id": 33, "links": [ ], "nullPointMode": "null as zero", "options": { diff --git a/operations/mimir-mixin-compiled-baremetal/dashboards/mimir-ruler.json b/operations/mimir-mixin-compiled-baremetal/dashboards/mimir-ruler.json index 3d48ecd7a9d..82c1eb169f1 100644 --- a/operations/mimir-mixin-compiled-baremetal/dashboards/mimir-ruler.json +++ b/operations/mimir-mixin-compiled-baremetal/dashboards/mimir-ruler.json @@ -1044,6 +1044,264 @@ "title": "Reads (ingesters)", "titleSize": "h6" }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash" + } + } + ] + } + ] + }, + "id": 11, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum by(instance) (rate(node_cpu_seconds_total{mode=\"user\",cluster=~\"$cluster\", namespace=~\"$namespace\",instance=~\".*ruler.*\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "{{instance}}", + "legendLink": null + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash" + } + } + ] + } + ] + }, + "id": 12, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",instance=~\".*ruler.*\"}\n- node_memory_MemFree_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",instance=~\".*ruler.*\"}\n- node_memory_Buffers_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",instance=~\".*ruler.*\"}\n- node_memory_Cached_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",instance=~\".*ruler.*\"}\n- node_memory_Slab_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",instance=~\".*ruler.*\"}\n- node_memory_PageTables_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",instance=~\".*ruler.*\"}\n- node_memory_SwapCached_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",instance=~\".*ruler.*\"}\n", + "format": "time_series", + "legendFormat": "{{instance}}", + "legendLink": null + } + ], + "title": "Memory (workingset)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "id": 13, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",instance=~\".*ruler.*\"})", + "format": "time_series", + "legendFormat": "{{instance}}", + "legendLink": null + } + ], + "title": "Memory (go heap inuse)", + "type": "timeseries" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Ruler resources", + "titleSize": "h6" + }, { "collapse": false, "height": "250px", @@ -1209,7 +1467,7 @@ } ] }, - "id": 11, + "id": 14, "links": [ ], "options": { "legend": { @@ -1257,7 +1515,7 @@ }, "overrides": [ ] }, - "id": 12, + "id": 15, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1348,7 +1606,7 @@ }, "overrides": [ ] }, - "id": 13, + "id": 16, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1427,7 +1685,7 @@ }, "overrides": [ ] }, - "id": 14, + "id": 17, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1524,7 +1782,7 @@ } ] }, - "id": 15, + "id": 18, "links": [ ], "options": { "legend": { @@ -1585,7 +1843,7 @@ }, "overrides": [ ] }, - "id": 16, + "id": 19, "links": [ ], "options": { "legend": { @@ -1634,7 +1892,7 @@ }, "overrides": [ ] }, - "id": 17, + "id": 20, "links": [ ], "options": { "legend": { @@ -1683,7 +1941,7 @@ }, "overrides": [ ] }, - "id": 18, + "id": 21, "links": [ ], "options": { "legend": { @@ -1743,7 +2001,7 @@ }, "overrides": [ ] }, - "id": 19, + "id": 22, "links": [ ], "options": { "legend": { @@ -1791,7 +2049,7 @@ }, "overrides": [ ] }, - "id": 20, + "id": 23, "links": [ ], "options": { "legend": { @@ -1839,7 +2097,7 @@ }, "overrides": [ ] }, - "id": 21, + "id": 24, "links": [ ], "options": { "legend": { @@ -1899,7 +2157,7 @@ }, "overrides": [ ] }, - "id": 22, + "id": 25, "links": [ ], "options": { "legend": { @@ -1959,7 +2217,7 @@ }, "overrides": [ ] }, - "id": 23, + "id": 26, "links": [ ], "options": { "legend": { @@ -1992,7 +2250,7 @@ "unit": "percentunit" } }, - "id": 24, + "id": 27, "links": [ ], "options": { "legend": { @@ -2040,7 +2298,7 @@ }, "overrides": [ ] }, - "id": 25, + "id": 28, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -2119,7 +2377,7 @@ }, "overrides": [ ] }, - "id": 26, + "id": 29, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -2210,7 +2468,7 @@ }, "overrides": [ ] }, - "id": 27, + "id": 30, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -2289,7 +2547,7 @@ }, "overrides": [ ] }, - "id": 28, + "id": 31, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -2368,7 +2626,7 @@ }, "overrides": [ ] }, - "id": 29, + "id": 32, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -2447,7 +2705,7 @@ }, "overrides": [ ] }, - "id": 30, + "id": 33, "links": [ ], "nullPointMode": "null as zero", "options": { diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-ruler.json b/operations/mimir-mixin-compiled/dashboards/mimir-ruler.json index 3d48ecd7a9d..0c1379cef55 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-ruler.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-ruler.json @@ -1044,6 +1044,288 @@ "title": "Reads (ingesters)", "titleSize": "h6" }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash" + } + } + ] + } + ] + }, + "id": 11, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})", + "format": "time_series", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\",resource=\"cpu\"})", + "format": "time_series", + "legendFormat": "request", + "legendLink": null + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash" + } + } + ] + } + ] + }, + "id": 12, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"} > 0)", + "format": "time_series", + "legendFormat": "limit", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\",resource=\"memory\"})", + "format": "time_series", + "legendFormat": "request", + "legendLink": null + } + ], + "title": "Memory (workingset)", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "id": 13, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + } + ], + "title": "Memory (go heap inuse)", + "type": "timeseries" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Ruler resources", + "titleSize": "h6" + }, { "collapse": false, "height": "250px", @@ -1209,7 +1491,7 @@ } ] }, - "id": 11, + "id": 14, "links": [ ], "options": { "legend": { @@ -1257,7 +1539,7 @@ }, "overrides": [ ] }, - "id": 12, + "id": 15, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1348,7 +1630,7 @@ }, "overrides": [ ] }, - "id": 13, + "id": 16, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1427,7 +1709,7 @@ }, "overrides": [ ] }, - "id": 14, + "id": 17, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1524,7 +1806,7 @@ } ] }, - "id": 15, + "id": 18, "links": [ ], "options": { "legend": { @@ -1585,7 +1867,7 @@ }, "overrides": [ ] }, - "id": 16, + "id": 19, "links": [ ], "options": { "legend": { @@ -1634,7 +1916,7 @@ }, "overrides": [ ] }, - "id": 17, + "id": 20, "links": [ ], "options": { "legend": { @@ -1683,7 +1965,7 @@ }, "overrides": [ ] }, - "id": 18, + "id": 21, "links": [ ], "options": { "legend": { @@ -1743,7 +2025,7 @@ }, "overrides": [ ] }, - "id": 19, + "id": 22, "links": [ ], "options": { "legend": { @@ -1791,7 +2073,7 @@ }, "overrides": [ ] }, - "id": 20, + "id": 23, "links": [ ], "options": { "legend": { @@ -1839,7 +2121,7 @@ }, "overrides": [ ] }, - "id": 21, + "id": 24, "links": [ ], "options": { "legend": { @@ -1899,7 +2181,7 @@ }, "overrides": [ ] }, - "id": 22, + "id": 25, "links": [ ], "options": { "legend": { @@ -1959,7 +2241,7 @@ }, "overrides": [ ] }, - "id": 23, + "id": 26, "links": [ ], "options": { "legend": { @@ -1992,7 +2274,7 @@ "unit": "percentunit" } }, - "id": 24, + "id": 27, "links": [ ], "options": { "legend": { @@ -2040,7 +2322,7 @@ }, "overrides": [ ] }, - "id": 25, + "id": 28, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -2119,7 +2401,7 @@ }, "overrides": [ ] }, - "id": 26, + "id": 29, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -2210,7 +2492,7 @@ }, "overrides": [ ] }, - "id": 27, + "id": 30, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -2289,7 +2571,7 @@ }, "overrides": [ ] }, - "id": 28, + "id": 31, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -2368,7 +2650,7 @@ }, "overrides": [ ] }, - "id": 29, + "id": 32, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -2447,7 +2729,7 @@ }, "overrides": [ ] }, - "id": 30, + "id": 33, "links": [ ], "nullPointMode": "null as zero", "options": { diff --git a/operations/mimir-mixin/dashboards/ruler.libsonnet b/operations/mimir-mixin/dashboards/ruler.libsonnet index 4c15d54e240..8d09e28806c 100644 --- a/operations/mimir-mixin/dashboards/ruler.libsonnet +++ b/operations/mimir-mixin/dashboards/ruler.libsonnet @@ -142,6 +142,18 @@ local filename = 'mimir-ruler.json'; $.latencyPanel('cortex_ingester_client_request_duration_seconds', '{%s, operation="/cortex.Ingester/QueryStream"}' % $.jobMatcher($._config.job_names.ruler + $._config.job_names.ruler_querier)) ) ) + .addRow( + $.row('Ruler resources') + .addPanel( + $.containerCPUUsagePanelByComponent('ruler'), + ) + .addPanel( + $.containerMemoryWorkingSetPanelByComponent('ruler'), + ) + .addPanel( + $.containerGoHeapInUsePanelByComponent('ruler'), + ) + ) .addRowIf( $._config.autoscaling.ruler.enabled, $.cpuAndMemoryBasedAutoScalingRow('Ruler'),