operator: Add support for configuring HTTP server timeouts (#9405)

Co-authored-by: Robert Jacob <rojacob@redhat.com>

operator/CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Main
 
+- [9405](https://github.com/grafana/loki/pull/9405) **periklis**: Add support for configuring HTTP server timeouts
 - [9378](https://github.com/grafana/loki/pull/9378) **aminesnow**: Add zone aware API spec validation
 - [9408](https://github.com/grafana/loki/pull/9408) **JoaoBraveCoding**: Add PodAntiAffinity overwrites per component
 - [9429](https://github.com/grafana/loki/pull/9429) **aminesnow**: Add default TopologySpreadContraints to Gateway

operator/apis/loki/v1/lokistack_types.go

@@ -561,7 +561,7 @@ type QueryLimitSpec struct {
 	//
 	// +optional
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:default:="1m"
+	// +kubebuilder:default:="3m"
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Query Timeout"
 	QueryTimeout string `json:"queryTimeout,omitempty"`
 }
@@ -913,6 +913,8 @@ const (
 	ReasonMissingGatewayOpenShiftBaseDomain LokiStackConditionReason = "MissingGatewayOpenShiftBaseDomain"
 	// ReasonFailedCertificateRotation when the reconciler cannot rotate any of the required TLS certificates.
 	ReasonFailedCertificateRotation LokiStackConditionReason = "FailedCertificateRotation"
+	// ReasonQueryTimeoutInvalid when the QueryTimeout can not be parsed.
+	ReasonQueryTimeoutInvalid LokiStackConditionReason = "ReasonQueryTimeoutInvalid"
 )
 
 // PodStatusMap defines the type for mapping pod status to pod name.

operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml

@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:main-ac1c1fd
-    createdAt: "2023-05-11T08:04:29Z"
+    createdAt: "2023-05-22T15:22:48Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     operators.operatorframework.io/builder: operator-sdk-unknown

operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml

@@ -151,7 +151,7 @@ spec:
                             format: int32
                             type: integer
                           queryTimeout:
-                            default: 1m
+                            default: 3m
                             description: Timeout when querying ingesters or storage
                               during the execution of a query request.
                             type: string
@@ -264,7 +264,7 @@ spec:
                               format: int32
                               type: integer
                             queryTimeout:
-                              default: 1m
+                              default: 3m
                               description: Timeout when querying ingesters or storage
                                 during the execution of a query request.
                               type: string

operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml

@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:main-ac1c1fd
-    createdAt: "2023-05-11T08:04:26Z"
+    createdAt: "2023-05-22T15:22:44Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     operators.operatorframework.io/builder: operator-sdk-unknown

operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml

@@ -151,7 +151,7 @@ spec:
                             format: int32
                             type: integer
                           queryTimeout:
-                            default: 1m
+                            default: 3m
                             description: Timeout when querying ingesters or storage
                               during the execution of a query request.
                             type: string
@@ -264,7 +264,7 @@ spec:
                               format: int32
                               type: integer
                             queryTimeout:
-                              default: 1m
+                              default: 3m
                               description: Timeout when querying ingesters or storage
                                 during the execution of a query request.
                               type: string

operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml

@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: quay.io/openshift-logging/loki-operator:v0.1.0
-    createdAt: "2023-05-11T08:04:32Z"
+    createdAt: "2023-05-22T15:22:53Z"
     description: |
       The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging.
       ## Prerequisites and Requirements

operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml

@@ -151,7 +151,7 @@ spec:
                             format: int32
                             type: integer
                           queryTimeout:
-                            default: 1m
+                            default: 3m
                             description: Timeout when querying ingesters or storage
                               during the execution of a query request.
                             type: string
@@ -264,7 +264,7 @@ spec:
                               format: int32
                               type: integer
                             queryTimeout:
-                              default: 1m
+                              default: 3m
                               description: Timeout when querying ingesters or storage
                                 during the execution of a query request.
                               type: string

operator/config/crd/bases/loki.grafana.com_lokistacks.yaml

@@ -134,7 +134,7 @@ spec:
                             format: int32
                             type: integer
                           queryTimeout:
-                            default: 1m
+                            default: 3m
                             description: Timeout when querying ingesters or storage
                               during the execution of a query request.
                             type: string
@@ -247,7 +247,7 @@ spec:
                               format: int32
                               type: integer
                             queryTimeout:
-                              default: 1m
+                              default: 3m
                               description: Timeout when querying ingesters or storage
                                 during the execution of a query request.
                               type: string

operator/docs/operator/api.md

@@ -1302,6 +1302,21 @@ the component onto it.</p>
 the component onto it.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>podAntiAffinity</code><br/>
+<em>
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#podantiaffinity-v1-core">
+Kubernetes core/v1.PodAntiAffinity
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>PodAntiAffinity defines the pod anti affinity scheduling rules to schedule pods
+of a component.</p>
+</td>
+</tr>
 </tbody>
 </table>
 
@@ -1553,6 +1568,9 @@ for the ruler is missing.</p>
 </tr><tr><td><p>&#34;PendingComponents&#34;</p></td>
 <td><p>ReasonPendingComponents when all/some LokiStack components pending dependencies</p>
 </td>
+</tr><tr><td><p>&#34;ReasonQueryTimeoutInvalid&#34;</p></td>
+<td><p>ReasonQueryTimeoutInvalid when the QueryTimeout can not be parsed.</p>
+</td>
 </tr><tr><td><p>&#34;ReadyComponents&#34;</p></td>
 <td><p>ReasonReadyComponents when all LokiStack components are ready to serve traffic.</p>
 </td>
@@ -3329,7 +3347,8 @@ int32
 </td>
 <td>
 <em>(Optional)</em>
-<p>Zones defines an array of ZoneSpec that the scheduler will try to satisfy.</p>
+<p>Zones defines an array of ZoneSpec that the scheduler will try to satisfy.
+IMPORTANT: Make sure that the replication factor defined is less than or equal to the number of available zones.</p>
 </td>
 </tr>
 </tbody>

operator/internal/handlers/lokistack_create_or_update.go

@@ -269,6 +269,16 @@ func CreateOrUpdateLokiStack(
 		certRotationRequiredAt = stack.Annotations[manifests.AnnotationCertRotationRequiredAt]
 	}
 
+	timeoutConfig, err := manifests.NewTimeoutConfig(stack.Spec.Limits)
+	if err != nil {
+		ll.Error(err, "failed to parse query timeout")
+		return &status.DegradedError{
+			Message: fmt.Sprintf("Error parsing query timeout: %s", err),
+			Reason:  lokiv1.ReasonQueryTimeoutInvalid,
+			Requeue: false,
+		}
+	}
+
 	// Here we will translate the lokiv1.LokiStack options into manifest options
 	opts := manifests.Options{
 		Name:                   req.Name,
@@ -286,6 +296,7 @@ func CreateOrUpdateLokiStack(
 			Spec:   rulerConfig,
 			Secret: rulerSecret,
 		},
+		Timeouts: timeoutConfig,
 		Tenants: manifests.Tenants{
 			Secrets: tenantSecrets,
 			Configs: tenantConfigs,

operator/internal/handlers/lokistack_create_or_update_test.go

@@ -1385,6 +1385,78 @@ func TestCreateOrUpdateLokiStack_MissingTenantsSpec_SetDegraded(t *testing.T) {
 	require.Equal(t, degradedErr, err)
 }
 
+func TestCreateOrUpdateLokiStack_WhenInvalidQueryTimeout_SetDegraded(t *testing.T) {
+	sw := &k8sfakes.FakeStatusWriter{}
+	k := &k8sfakes.FakeClient{}
+	r := ctrl.Request{
+		NamespacedName: types.NamespacedName{
+			Name:      "my-stack",
+			Namespace: "some-ns",
+		},
+	}
+
+	degradedErr := &status.DegradedError{
+		Message: `Error parsing query timeout: time: invalid duration "invalid"`,
+		Reason:  lokiv1.ReasonQueryTimeoutInvalid,
+		Requeue: false,
+	}
+
+	stack := &lokiv1.LokiStack{
+		TypeMeta: metav1.TypeMeta{
+			Kind: "LokiStack",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "my-stack",
+			Namespace: "some-ns",
+			UID:       "b23f9a38-9672-499f-8c29-15ede74d3ece",
+		},
+		Spec: lokiv1.LokiStackSpec{
+			Size: lokiv1.SizeOneXExtraSmall,
+			Storage: lokiv1.ObjectStorageSpec{
+				Schemas: []lokiv1.ObjectStorageSchema{
+					{
+						Version:       lokiv1.ObjectStorageSchemaV12,
+						EffectiveDate: "2023-05-22",
+					},
+				},
+				Secret: lokiv1.ObjectStorageSecretSpec{
+					Name: defaultSecret.Name,
+					Type: lokiv1.ObjectStorageSecretS3,
+				},
+			},
+			Tenants: &lokiv1.TenantsSpec{
+				Mode: "openshift",
+			},
+			Limits: &lokiv1.LimitsSpec{
+				Global: &lokiv1.LimitsTemplateSpec{
+					QueryLimits: &lokiv1.QueryLimitSpec{
+						QueryTimeout: "invalid",
+					},
+				},
+			},
+		},
+	}
+
+	// Create looks up the CR first, so we need to return our fake stack
+	k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error {
+		if r.Name == name.Name && r.Namespace == name.Namespace {
+			k.SetClientObject(object, stack)
+		}
+		if defaultSecret.Name == name.Name {
+			k.SetClientObject(object, &defaultSecret)
+		}
+		return nil
+	}
+
+	k.StatusStub = func() client.StatusWriter { return sw }
+
+	err := handlers.CreateOrUpdateLokiStack(context.TODO(), logger, r, k, scheme, featureGates)
+
+	// make sure error is returned
+	require.Error(t, err)
+	require.Equal(t, degradedErr, err)
+}
+
 func TestCreateOrUpdateLokiStack_RemovesRulerResourcesWhenDisabled(t *testing.T) {
 	sw := &k8sfakes.FakeStatusWriter{}
 	k := &k8sfakes.FakeClient{}

operator/internal/manifests/build_test.go

@@ -36,6 +36,7 @@ func TestApplyUserOptions_OverrideDefaults(t *testing.T) {
 					},
 				},
 			},
+			Timeouts: defaultTimeoutConfig,
 		}
 		err := ApplyDefaultSettings(&opt)
 		defs := internal.StackSizeTable[size]
@@ -78,6 +79,7 @@ func TestApplyUserOptions_AlwaysSetCompactorReplicasToOne(t *testing.T) {
 					},
 				},
 			},
+			Timeouts: defaultTimeoutConfig,
 		}
 		err := ApplyDefaultSettings(&opt)
 		defs := internal.StackSizeTable[size]
@@ -232,6 +234,7 @@ func TestBuildAll_WithFeatureGates_ServiceMonitors(t *testing.T) {
 						ServingCertsService: false,
 					},
 				},
+				Timeouts: defaultTimeoutConfig,
 			},
 		},
 		{
@@ -250,6 +253,7 @@ func TestBuildAll_WithFeatureGates_ServiceMonitors(t *testing.T) {
 						ServingCertsService: false,
 					},
 				},
+				Timeouts: defaultTimeoutConfig,
 			},
 		},
 	}
@@ -292,6 +296,7 @@ func TestBuildAll_WithFeatureGates_OpenShift_ServingCertsService(t *testing.T) {
 						ServingCertsService: false,
 					},
 				},
+				Timeouts: defaultTimeoutConfig,
 			},
 		},
 		{
@@ -309,6 +314,7 @@ func TestBuildAll_WithFeatureGates_OpenShift_ServingCertsService(t *testing.T) {
 						ServingCertsService: true,
 					},
 				},
+				Timeouts: defaultTimeoutConfig,
 			},
 		},
 	}
@@ -349,6 +355,7 @@ func TestBuildAll_WithFeatureGates_HTTPEncryption(t *testing.T) {
 		Gates: configv1.FeatureGates{
 			HTTPEncryption: true,
 		},
+		Timeouts: defaultTimeoutConfig,
 	}
 
 	err := ApplyDefaultSettings(&opts)
@@ -422,6 +429,7 @@ func TestBuildAll_WithFeatureGates_ServiceMonitorTLSEndpoints(t *testing.T) {
 			HTTPEncryption:             true,
 			ServiceMonitorTLSEndpoints: true,
 		},
+		Timeouts: defaultTimeoutConfig,
 	}
 
 	err := ApplyDefaultSettings(&opts)
@@ -526,6 +534,7 @@ func TestBuildAll_WithFeatureGates_GRPCEncryption(t *testing.T) {
 				Gates: configv1.FeatureGates{
 					GRPCEncryption: false,
 				},
+				Timeouts: defaultTimeoutConfig,
 			},
 		},
 		{
@@ -568,6 +577,7 @@ func TestBuildAll_WithFeatureGates_GRPCEncryption(t *testing.T) {
 				Gates: configv1.FeatureGates{
 					GRPCEncryption: true,
 				},
+				Timeouts: defaultTimeoutConfig,
 			},
 		},
 	}
@@ -692,6 +702,7 @@ func TestBuildAll_WithFeatureGates_RuntimeSeccompProfile(t *testing.T) {
 				Gates: configv1.FeatureGates{
 					RuntimeSeccompProfile: false,
 				},
+				Timeouts: defaultTimeoutConfig,
 			},
 		},
 		{
@@ -734,6 +745,7 @@ func TestBuildAll_WithFeatureGates_RuntimeSeccompProfile(t *testing.T) {
 				Gates: configv1.FeatureGates{
 					RuntimeSeccompProfile: true,
 				},
+				Timeouts: defaultTimeoutConfig,
 			},
 		},
 	}
@@ -797,6 +809,7 @@ func TestBuildAll_WithFeatureGates_LokiStackGateway(t *testing.T) {
 					HTTPEncryption:             true,
 					ServiceMonitorTLSEndpoints: false,
 				},
+				Timeouts: defaultTimeoutConfig,
 			},
 		},
 		{
@@ -835,6 +848,7 @@ func TestBuildAll_WithFeatureGates_LokiStackGateway(t *testing.T) {
 					HTTPEncryption:             true,
 					ServiceMonitorTLSEndpoints: true,
 				},
+				Timeouts: defaultTimeoutConfig,
 			},
 		},
 	}
@@ -873,6 +887,7 @@ func TestBuildAll_WithFeatureGates_LokiStackAlerts(t *testing.T) {
 					ServiceMonitors: false,
 					LokiStackAlerts: false,
 				},
+				Timeouts: defaultTimeoutConfig,
 			},
 		},
 		{
@@ -887,6 +902,7 @@ func TestBuildAll_WithFeatureGates_LokiStackAlerts(t *testing.T) {
 					ServiceMonitors: true,
 					LokiStackAlerts: true,
 				},
+				Timeouts: defaultTimeoutConfig,
 			},
 		},
 	}