feat: Assert for bad max_query_lookback config (#16362)

**What this PR does / why we need it**: Add a new assert to our jsonnet manifest that checks that there's no tenant with a retention period that can't be queried (due to low max_query_lookback)
grafana · Feb 19, 2025 · 073c94c · 073c94c
1 parent 6abb12d
commit 073c94c
Show file tree

Hide file tree

Showing 2 changed files with 56 additions and 1 deletion.
diff --git a/production/ksonnet/loki/common.libsonnet b/production/ksonnet/loki/common.libsonnet
@@ -44,6 +44,18 @@ local k = import 'ksonnet-util/kausal.libsonnet';
       container.mixin.readinessProbe.httpGet.withPort($._config.http_listen_port) +
       container.mixin.readinessProbe.withInitialDelaySeconds(15) +
       container.mixin.readinessProbe.withTimeoutSeconds(1),
+
+    parseDuration(duration)::
+      if std.endsWith(duration, 's') then
+        std.parseInt(std.substr(duration, 0, std.length(duration) - 1))
+      else if std.endsWith(duration, 'm') then
+        std.parseInt(std.substr(duration, 0, std.length(duration) - 1)) * 60
+      else if std.endsWith(duration, 'h') then
+        std.parseInt(std.substr(duration, 0, std.length(duration) - 1)) * 3600
+      else if std.endsWith(duration, 'd') then
+        std.parseInt(std.substr(duration, 0, std.length(duration) - 1)) * 86400
+      else
+        error 'unable to parse duration %s' % duration,
   },
 
   // functions for k8s objects

diff --git a/production/ksonnet/loki/overrides.libsonnet b/production/ksonnet/loki/overrides.libsonnet
@@ -1,5 +1,5 @@
+local common = import 'common.libsonnet';
 local k = import 'ksonnet-util/kausal.libsonnet';
-
 {
   _config+: {
     overrides: {
@@ -39,4 +39,47 @@ local k = import 'ksonnet-util/kausal.libsonnet';
         + (if std.length($._config.multi_kv_config) > 0 then { multi_kv_config: $._config.multi_kv_config } else {}),
       ),
     }),
+
+  local checkRetentionStreams(retentionStreams, maxQueryLookback) =
+    std.foldl(
+      function(acc, retentionStream) acc && common.parseDuration(retentionStream.period) <= common.parseDuration(maxQueryLookback),
+      retentionStreams,
+      true
+    ),
+
+  isLookbackLongerThanRetention(tenantCfg)::
+    local retentionPeriod = tenantCfg.retention_period;
+    local lookback = tenantCfg.max_query_lookback;
+    if std.objectHas(tenantCfg, 'max_query_lookback') &&
+       std.objectHas(tenantCfg, 'retention_period') then
+      common.parseDuration(lookback) >= common.parseDuration(retentionPeriod)
+    else
+      true,
+
+  isLookbackLongerThanStreamRetention(tenantCfg)::
+    local retentionStream = tenantCfg.retention_stream;
+    local lookback = tenantCfg.max_query_lookback;
+    if std.objectHas(tenantCfg, 'max_query_lookback') &&
+       std.objectHas(tenantCfg, 'retention_stream') then
+      checkRetentionStreams(retentionStream, lookback)
+    else
+      true,
+
+  checkTenantRetention(tenant)::
+    local tenantCfg = $._config.overrides[tenant];
+    if $.isLookbackLongerThanRetention(tenantCfg) &&
+       $.isLookbackLongerThanStreamRetention(tenantCfg) then
+      true
+    else
+      false,
+
+  local tenants = std.objectFields($._config.overrides),
+
+  local validRetentionsCheck = std.foldl(
+    function(acc, tenant) if !$.checkTenantRetention(tenant) then { valid: false, failedTenant: [tenant] + acc.failedTenant } else acc,
+    tenants,
+    { valid: true, failedTenant: [] }
+  ),
+
+  assert validRetentionsCheck.valid : 'retention period longer than max_query_lookback for tenants %s' % std.join(', ', validRetentionsCheck.failedTenant),
 }