Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

operator: Add support for configuring HTTP server timeouts #9405

Merged
merged 35 commits into from
May 22, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
70d3ea6
operator: Add LokiStack http server limits spec
periklis May 5, 2023
5932c37
Parse server config options from limits spec
periklis May 5, 2023
f5eb750
Add OpenShift route timeout annotation
periklis May 5, 2023
643fc08
Pass server options to loki config
periklis May 5, 2023
e9aaf56
Fix defaults
periklis May 5, 2023
c3a6557
Apply http server and upstream timeouts to the gateway
periklis May 5, 2023
268daaa
Add changelog entry
periklis May 5, 2023
d19eb9d
Add api docs
periklis May 5, 2023
89c7465
Merge branch 'main' into operator-fix-write-timeouts
periklis May 8, 2023
a37e1a2
Add operator-sdk API markers
periklis May 8, 2023
11c531e
Enhance error log message in lokistack handler
periklis May 8, 2023
1c661c5
Merge branch 'main' into operator-fix-write-timeouts
periklis May 8, 2023
08ae251
Merge branch 'main' into operator-fix-write-timeouts
periklis May 9, 2023
9079100
Apply code review suggestions
periklis May 15, 2023
2723949
Merge remote-tracking branch 'upstream' into operator-fix-write-timeouts
periklis May 15, 2023
146a034
Fix merge
periklis May 15, 2023
0bff0e9
Merge remote-tracking branch 'upstream' into operator-fix-write-timeouts
periklis May 17, 2023
10f668e
Regenerate
periklis May 17, 2023
d5e0bbb
Make defaults a global variable, remove pointer
xperimental May 17, 2023
d5d883d
Make defaults more visible in parse function
xperimental May 17, 2023
42e5b49
Unify defaults and simplify NewServerConfig
xperimental May 17, 2023
642dfba
Rename constants
xperimental May 17, 2023
094a59d
Use time.Duration in config template
xperimental May 17, 2023
cd9731d
Reduce indirection in config generator
xperimental May 17, 2023
a732ece
Separate config structs for Loki and gateway
xperimental May 17, 2023
9c4bb0b
Rename configuration struct
xperimental May 17, 2023
47d56cc
Merge remote-tracking branch 'upstream' into operator-fix-write-timeouts
periklis May 17, 2023
12e6080
Rename TestNewServerConfig to TestNewTimeoutConfig
periklis May 17, 2023
c441848
Raise error instead of using defaults
xperimental May 22, 2023
3382fac
Shorten gateway route timeout
xperimental May 22, 2023
4cb456c
Increase default query timeout to match existing default
xperimental May 22, 2023
11d0ebf
Regenerate docs
periklis May 22, 2023
43be4ef
Merge remote-tracking branch 'upstream' into operator-fix-write-timeouts
periklis May 22, 2023
38fc277
Fix failing tests
periklis May 22, 2023
7a2d0d2
Update import formatting to satisfy check
xperimental May 22, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Unify defaults and simplify NewServerConfig
  • Loading branch information
xperimental authored and periklis committed May 17, 2023
commit 42e5b495fb531b79d50ac0e5da6624092b1cd6cc
6 changes: 3 additions & 3 deletions operator/internal/manifests/gateway_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -791,11 +791,11 @@ func TestBuildGateway_WithHTTPEncryption(t *testing.T) {
"--logs.read.endpoint=https://abcd-query-frontend-http.efgh.svc.cluster.local:3100",
"--logs.tail.endpoint=https://abcd-query-frontend-http.efgh.svc.cluster.local:3100",
"--logs.write.endpoint=https://abcd-distributor-http.efgh.svc.cluster.local:3100",
"--logs.write-timeout=10m0s",
"--logs.write-timeout=2m0s",
"--rbac.config=/etc/lokistack-gateway/rbac.yaml",
"--tenants.config=/etc/lokistack-gateway/tenants.yaml",
"--server.read-timeout=1m0s",
"--server.write-timeout=12m0s",
"--server.read-timeout=36s",
"--server.write-timeout=4m0s",
"--logs.rules.endpoint=https://abcd-ruler-http.efgh.svc.cluster.local:3100",
"--logs.rules.read-only=true",
"--tls.client-auth-type=NoClientCert",
Expand Down
35 changes: 13 additions & 22 deletions operator/internal/manifests/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"github.com/grafana/loki/operator/internal/manifests/internal"
"github.com/grafana/loki/operator/internal/manifests/openshift"
"github.com/grafana/loki/operator/internal/manifests/storage"
"github.com/imdario/mergo"
)

// Options is a set of configuration values to use when building manifests such as resource sizes, etc.
Expand Down Expand Up @@ -137,39 +136,38 @@ func NewServerConfig(s *lokiv1.LimitsSpec) (ServerConfig, error) {
return defaultServerConfig, nil
}

var (
maxTenantQueryTimeout time.Duration
globalQueryTimeout time.Duration
)

queryTimeout := lokiDefaultQueryTimeout
if s.Global.QueryLimits != nil && s.Global.QueryLimits.QueryTimeout != "" {
var err error
globalQueryTimeout, err = time.ParseDuration(s.Global.QueryLimits.QueryTimeout)
globalQueryTimeout, err := time.ParseDuration(s.Global.QueryLimits.QueryTimeout)
if err != nil {
return ServerConfig{}, err
}

if globalQueryTimeout > queryTimeout {
queryTimeout = globalQueryTimeout
}
}

for _, tLimit := range s.Tenants {
if tLimit.QueryLimits == nil || tLimit.QueryLimits.QueryTimeout == "" {
continue
}

tQueryTimeout, err := time.ParseDuration(tLimit.QueryLimits.QueryTimeout)
tenantQueryTimeout, err := time.ParseDuration(tLimit.QueryLimits.QueryTimeout)
if err != nil {
return ServerConfig{}, err
}

if maxTenantQueryTimeout < tQueryTimeout {
maxTenantQueryTimeout = tQueryTimeout
if tenantQueryTimeout > queryTimeout {
queryTimeout = tenantQueryTimeout
}
}

queryTimeout := maxTenantQueryTimeout
if queryTimeout < globalQueryTimeout {
queryTimeout = globalQueryTimeout
}
return calculateHTTPTimeouts(queryTimeout), nil
}

func calculateHTTPTimeouts(queryTimeout time.Duration) ServerConfig {
idleTimeout := lokiDefaultHTTPIdleTimeout
if queryTimeout < idleTimeout {
idleTimeout = queryTimeout
Expand All @@ -178,8 +176,7 @@ func NewServerConfig(s *lokiv1.LimitsSpec) (ServerConfig, error) {
readTimeout := queryTimeout / 10
writeTimeout := queryTimeout + lokiQueryTimeoutTimeoutWiggleRoom

defaults := defaultServerConfig
customCfg := ServerConfig{
return ServerConfig{
HTTP: HTTPConfig{
IdleTimeout: idleTimeout,
ReadTimeout: readTimeout,
Expand All @@ -189,10 +186,4 @@ func NewServerConfig(s *lokiv1.LimitsSpec) (ServerConfig, error) {
GatewayUpstreamWriteTimeout: writeTimeout,
},
}

if err := mergo.Merge(&defaults, &customCfg, mergo.WithOverride); err != nil {
return defaults, err
}

return defaults, nil
}
20 changes: 4 additions & 16 deletions operator/internal/manifests/var.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,28 +111,16 @@ const (
)

const (
lokiDefaultQueryTimeout = 1 * time.Minute
lokiDefaultHTTPIdleTimeout = 30 * time.Second
lokiDefaultHTTPReadTimeout = 30 * time.Second
lokiDefaultHTTPWriteTimeout = 10 * time.Minute
lokiQueryTimeoutTimeoutWiggleRoom = 1 * time.Minute

gatewayReadWiggleRoom = 30 * time.Second
gatewayWriteWiggleRoom = 2 * time.Minute
gatewayDefaultReadTimeout = lokiDefaultHTTPReadTimeout + gatewayReadWiggleRoom
gatewayDefaultWriteTimeout = lokiDefaultHTTPWriteTimeout + gatewayWriteWiggleRoom
gatewayReadWiggleRoom = 30 * time.Second
gatewayWriteWiggleRoom = 2 * time.Minute
)

var (
defaultServerConfig = ServerConfig{
HTTP: HTTPConfig{
IdleTimeout: lokiDefaultHTTPIdleTimeout,
ReadTimeout: lokiDefaultHTTPReadTimeout,
WriteTimeout: lokiDefaultHTTPWriteTimeout,
GatewayReadTimeout: gatewayDefaultReadTimeout,
GatewayWriteTimeout: gatewayDefaultWriteTimeout,
GatewayUpstreamWriteTimeout: lokiDefaultHTTPWriteTimeout,
},
}
defaultServerConfig = calculateHTTPTimeouts(lokiDefaultQueryTimeout)

defaultConfigMapMode = int32(420)
volumeFileSystemMode = corev1.PersistentVolumeFilesystem
Expand Down