Skip to content

Commit

Permalink
render: data timeout for history queries
Browse files Browse the repository at this point in the history
  • Loading branch information
msaf1980 committed Mar 22, 2022
1 parent 27599e8 commit 40330ca
Show file tree
Hide file tree
Showing 7 changed files with 295 additions and 14 deletions.
58 changes: 56 additions & 2 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"net/url"
"os"
"regexp"
"sort"
"strings"
"time"

Expand Down Expand Up @@ -67,10 +68,41 @@ var IndexReverse = map[string]uint8{
// IndexReverseNames contains valid names for index-reverse setting
var IndexReverseNames = []string{"auto", "direct", "reversed"}

type QueryParam struct {
Duration time.Duration `toml:"duration" json:"duration" comment:"minimal duration (beetween from/until) for select query params"`
URL string `toml:"url" json:"url" comment:"url for queries with durations greater or equal than "`
DataTimeout time.Duration `toml:"data-timeout" json:"data-timeout" comment:"total timeout to fetch data"`
}

func binarySearchQueryParam(a []QueryParam, duration time.Duration, start, end int) int {
length := end - start
if length <= 0 {
return -1 // not found
}

var result int
mid := start + length/2
if a[mid].Duration > duration {
result = binarySearchQueryParam(a, duration, start, mid)
} else {
if result = binarySearchQueryParam(a, duration, mid+1, end); result == -1 {
result = mid
}
}

return result
}

// search on sorted slice
func BinarySearchQueryParam(a []QueryParam, duration time.Duration) int {
return binarySearchQueryParam(a, duration, 0, len(a))
}

// ClickHouse config
type ClickHouse struct {
URL string `toml:"url" json:"url" comment:"see https://clickhouse.tech/docs/en/interfaces/http"`
DataTimeout time.Duration `toml:"data-timeout" json:"data-timeout" comment:"total timeout to fetch data"`
URL string `toml:"url" json:"url" comment:"default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params"`
DataTimeout time.Duration `toml:"data-timeout" json:"data-timeout" comment:"default total timeout to fetch data, can be overwritten with query-params"`
QueryParams []QueryParam `toml:"query-params" json:"query-params" comment:"customized query params (url, data timeout) for durations greater or equal"`
IndexTable string `toml:"index-table" json:"index-table" comment:"see doc/index-table.md"`
IndexUseDaily bool `toml:"index-use-daily" json:"index-use-daily"`
IndexReverse string `toml:"index-reverse" json:"index-reverse" comment:"see doc/config.md"`
Expand Down Expand Up @@ -340,6 +372,28 @@ func Unmarshal(body []byte) (*Config, error) {
cfg.Logging = make([]zapwriter.Config, 0)
}

sort.SliceStable(cfg.ClickHouse.QueryParams, func(i, j int) bool {
return cfg.ClickHouse.QueryParams[i].Duration > cfg.ClickHouse.QueryParams[j].Duration
})

for i := range cfg.ClickHouse.QueryParams {
if cfg.ClickHouse.QueryParams[i].Duration == 0 {
return nil, fmt.Errorf("query duration param not set for: %+v", cfg.ClickHouse.QueryParams[i])
}
if cfg.ClickHouse.QueryParams[i].DataTimeout == 0 {
return nil, fmt.Errorf("query data-timeout param not set for: %+v", cfg.ClickHouse.QueryParams[i])
}
if cfg.ClickHouse.QueryParams[i].URL == "" {
// reuse default url
cfg.ClickHouse.QueryParams[i].URL = cfg.ClickHouse.URL
}
}

cfg.ClickHouse.QueryParams = append(
[]QueryParam{{URL: cfg.ClickHouse.URL, DataTimeout: cfg.ClickHouse.DataTimeout}},
cfg.ClickHouse.QueryParams...,
)

if len(cfg.Logging) == 0 {
cfg.Logging = append(cfg.Logging, newLoggingConfig())
}
Expand Down
11 changes: 9 additions & 2 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -327,8 +327,15 @@ sample-thereafter = 12

// ClickHouse
expected.ClickHouse = ClickHouse{
URL: "http://somehost:8123",
DataTimeout: 64000000000,
URL: "http://somehost:8123",
DataTimeout: 64000000000,
QueryParams: []QueryParam{
{
Duration: 0,
URL: "http://somehost:8123",
DataTimeout: 64000000000,
},
},
IndexTable: "graphite_index",
IndexReverse: "direct",
IndexReverses: make(IndexReverses, 2),
Expand Down
17 changes: 17 additions & 0 deletions deploy/doc/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,23 @@ Useless settings:
- `max_ast_elements`: the same
- `max_execution_time`: with `cancel_http_readonly_queries_on_client_close=1` and `data-timeout = "1m"` it's already covered.

### Query multi parameters (for overwrite default url and data-timeout)

For queries with duration (until - from) >= 72 hours, use custom url and data-timeout

`
url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=102400000&max_result_bytes=12800000&max_threads=2"
data-timeout = "30s"

query-params = [
{
duration = "72h",
url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=1024000000&max_result_bytes=128000000&max_threads=1",
data-timeout = "60s"
}
]
`

### Index table
See [index table](./index-table.md) documentation for details.

Expand Down
21 changes: 19 additions & 2 deletions doc/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,23 @@ Useless settings:
- `max_ast_elements`: the same
- `max_execution_time`: with `cancel_http_readonly_queries_on_client_close=1` and `data-timeout = "1m"` it's already covered.

### Query multi parameters (for overwrite default url and data-timeout)

For queries with duration (until - from) >= 72 hours, use custom url and data-timeout

`
url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=102400000&max_result_bytes=12800000&max_threads=2"
data-timeout = "30s"

query-params = [
{
duration = "72h",
url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=1024000000&max_result_bytes=128000000&max_threads=1",
data-timeout = "60s"
}
]
`

### Index table
See [index table](./index-table.md) documentation for details.

Expand Down Expand Up @@ -112,9 +129,9 @@ It's possible to set multiple loggers. See `Config` description in [config.go](h
headers-to-log = []

[clickhouse]
# see https://clickhouse.tech/docs/en/interfaces/http
# default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params
url = "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1"
# total timeout to fetch data
# default total timeout to fetch data, can be overwritten with query-params
data-timeout = "1m0s"
# see doc/index-table.md
index-table = "graphite_index"
Expand Down
25 changes: 24 additions & 1 deletion render/data/multi_target.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"sync"
"time"

v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
"github.com/lomik/graphite-clickhouse/config"
Expand Down Expand Up @@ -58,6 +59,26 @@ func (m *MultiTarget) checkMetricsLimitExceeded(num int) error {
return nil
}

func getDataTimeout(cfg *config.Config, m *MultiTarget) time.Duration {
dataTimeout := cfg.ClickHouse.DataTimeout
if len(cfg.ClickHouse.QueryParams) > 1 {
var maxDuration time.Duration
for tf, _ := range *m {
duration := time.Second * time.Duration(tf.Until-tf.From)
if duration >= maxDuration {
maxDuration = duration
}
}

i := config.BinarySearchQueryParam(cfg.ClickHouse.QueryParams, maxDuration)
if i >= 0 {
return cfg.ClickHouse.QueryParams[i].DataTimeout
}
}

return dataTimeout
}

// Fetch fetches the parsed ClickHouse data returns CHResponses
func (m *MultiTarget) Fetch(ctx context.Context, cfg *config.Config, chContext string) (CHResponses, error) {
var lock sync.RWMutex
Expand All @@ -71,7 +92,9 @@ func (m *MultiTarget) Fetch(ctx context.Context, cfg *config.Config, chContext s
return nil, err
}

ctxTimeout, cancel := context.WithTimeout(ctx, cfg.ClickHouse.DataTimeout)
dataTimeout := getDataTimeout(cfg, m)

ctxTimeout, cancel := context.WithTimeout(ctx, dataTimeout)
defer cancel()

errors := make([]error, 0, len(*m))
Expand Down
154 changes: 154 additions & 0 deletions render/data/multi_target_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
package data

import (
"testing"
"time"

"github.com/lomik/graphite-clickhouse/config"
)

func Test_getDataTimeout(t *testing.T) {
tests := []struct {
name string
cfg *config.Config
m *MultiTarget
want time.Duration
}{
{
name: "one DataTimeout",
cfg: &config.Config{
ClickHouse: config.ClickHouse{
DataTimeout: time.Second,
QueryParams: []config.QueryParam{
{ // default params
Duration: 0,
DataTimeout: time.Second,
},
},
},
},
m: &MultiTarget{
TimeFrame{
From: 1647198000,
Until: 1647234000,
}: &Targets{},
},
want: time.Second,
},
{
name: "default DataTimeout",
cfg: &config.Config{
ClickHouse: config.ClickHouse{
DataTimeout: time.Second,
QueryParams: []config.QueryParam{
{ // default params
Duration: 0,
DataTimeout: time.Second,
},
{
Duration: time.Hour,
DataTimeout: time.Minute,
},
},
},
},
m: &MultiTarget{
TimeFrame{ // 1 hour - 1s
From: 1647198000,
Until: 1647201600 - 1,
}: &Targets{},
},
want: time.Second,
},
{
name: "1m DataTimeout (1 param), select 1h duration",
cfg: &config.Config{
ClickHouse: config.ClickHouse{
DataTimeout: time.Second * 10,
QueryParams: []config.QueryParam{
{ // default params
Duration: 0,
DataTimeout: time.Second,
},
{
Duration: time.Hour,
DataTimeout: time.Minute,
},
},
},
},
m: &MultiTarget{
TimeFrame{ // 1 hour
From: 1647198000,
Until: 1647201600,
}: &Targets{},
},
want: time.Minute,
},
{
name: "1m DataTimeout (2 param), select 1h duration",
cfg: &config.Config{
ClickHouse: config.ClickHouse{
DataTimeout: time.Second,
QueryParams: []config.QueryParam{
{ // default params
Duration: 0,
DataTimeout: time.Second,
},
{
Duration: time.Hour,
DataTimeout: time.Minute,
},
{
Duration: time.Hour * 2,
DataTimeout: 10 * time.Minute,
},
},
},
},
m: &MultiTarget{
TimeFrame{ // 1 hour
From: 1647198000,
Until: 1647201600,
}: &Targets{},
},
want: time.Minute,
},
{
name: "10m DataTimeout (2 param), select 2h1s duration",
cfg: &config.Config{
ClickHouse: config.ClickHouse{
DataTimeout: time.Second,
QueryParams: []config.QueryParam{
{ // default params
Duration: 0,
DataTimeout: time.Second,
},
{
Duration: time.Hour,
DataTimeout: time.Minute,
},
{
Duration: time.Hour * 2,
DataTimeout: 10 * time.Minute,
},
},
},
},
m: &MultiTarget{
TimeFrame{ // 2 hour 1s
From: 1647198000,
Until: 1647205201,
}: &Targets{},
},
want: 10 * time.Minute,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := getDataTimeout(tt.cfg, tt.m); got != tt.want {
t.Errorf("getDataTimeout() = %v, want %v", got, tt.want)
}
})
}
}
Loading

0 comments on commit 40330ca

Please sign in to comment.