Skip to content

Commit

Permalink
render: data timeout for history queries
Browse files Browse the repository at this point in the history
  • Loading branch information
msaf1980 committed Mar 23, 2022
1 parent 518d28a commit c5000f1
Show file tree
Hide file tree
Showing 7 changed files with 301 additions and 18 deletions.
58 changes: 56 additions & 2 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"net/url"
"os"
"regexp"
"sort"
"strings"
"time"

Expand Down Expand Up @@ -67,10 +68,41 @@ var IndexReverse = map[string]uint8{
// IndexReverseNames contains valid names for index-reverse setting
var IndexReverseNames = []string{"auto", "direct", "reversed"}

type QueryParam struct {
Duration time.Duration `toml:"duration" json:"duration" comment:"minimal duration (beetween from/until) for select query params"`
URL string `toml:"url" json:"url" comment:"url for queries with durations greater or equal than "`
DataTimeout time.Duration `toml:"data-timeout" json:"data-timeout" comment:"total timeout to fetch data"`
}

func binarySearchQueryParam(a []QueryParam, duration time.Duration, start, end int) int {
length := end - start
if length <= 0 {
return -1 // not found
}

var result int
mid := start + length/2
if a[mid].Duration > duration {
result = binarySearchQueryParam(a, duration, start, mid)
} else {
if result = binarySearchQueryParam(a, duration, mid+1, end); result == -1 {
result = mid
}
}

return result
}

// search on sorted slice
func BinarySearchQueryParam(a []QueryParam, duration time.Duration) int {
return binarySearchQueryParam(a, duration, 0, len(a))
}

// ClickHouse config
type ClickHouse struct {
URL string `toml:"url" json:"url" comment:"see https://clickhouse.tech/docs/en/interfaces/http"`
DataTimeout time.Duration `toml:"data-timeout" json:"data-timeout" comment:"total timeout to fetch data"`
URL string `toml:"url" json:"url" comment:"default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params"`
DataTimeout time.Duration `toml:"data-timeout" json:"data-timeout" comment:"default total timeout to fetch data, can be overwritten with query-params"`
QueryParams []QueryParam `toml:"query-params" json:"query-params" comment:"customized query params (url, data timeout) for durations greater or equal"`
IndexTable string `toml:"index-table" json:"index-table" comment:"see doc/index-table.md"`
IndexUseDaily bool `toml:"index-use-daily" json:"index-use-daily"`
IndexReverse string `toml:"index-reverse" json:"index-reverse" comment:"see doc/config.md"`
Expand Down Expand Up @@ -340,6 +372,28 @@ func Unmarshal(body []byte) (*Config, error) {
cfg.Logging = make([]zapwriter.Config, 0)
}

sort.SliceStable(cfg.ClickHouse.QueryParams, func(i, j int) bool {
return cfg.ClickHouse.QueryParams[i].Duration > cfg.ClickHouse.QueryParams[j].Duration
})

for i := range cfg.ClickHouse.QueryParams {
if cfg.ClickHouse.QueryParams[i].Duration == 0 {
return nil, fmt.Errorf("query duration param not set for: %+v", cfg.ClickHouse.QueryParams[i])
}
if cfg.ClickHouse.QueryParams[i].DataTimeout == 0 {
return nil, fmt.Errorf("query data-timeout param not set for: %+v", cfg.ClickHouse.QueryParams[i])
}
if cfg.ClickHouse.QueryParams[i].URL == "" {
// reuse default url
cfg.ClickHouse.QueryParams[i].URL = cfg.ClickHouse.URL
}
}

cfg.ClickHouse.QueryParams = append(
[]QueryParam{{URL: cfg.ClickHouse.URL, DataTimeout: cfg.ClickHouse.DataTimeout}},
cfg.ClickHouse.QueryParams...,
)

if len(cfg.Logging) == 0 {
cfg.Logging = append(cfg.Logging, newLoggingConfig())
}
Expand Down
11 changes: 9 additions & 2 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -327,8 +327,15 @@ sample-thereafter = 12

// ClickHouse
expected.ClickHouse = ClickHouse{
URL: "http://somehost:8123",
DataTimeout: 64000000000,
URL: "http://somehost:8123",
DataTimeout: 64000000000,
QueryParams: []QueryParam{
{
Duration: 0,
URL: "http://somehost:8123",
DataTimeout: 64000000000,
},
},
IndexTable: "graphite_index",
IndexReverse: "direct",
IndexReverses: make(IndexReverses, 2),
Expand Down
22 changes: 20 additions & 2 deletions deploy/doc/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,23 @@ Useless settings:
- `max_ast_elements`: the same
- `max_execution_time`: with `cancel_http_readonly_queries_on_client_close=1` and `data-timeout = "1m"` it's already covered.

### Query multi parameters (for overwrite default url and data-timeout)

For queries with duration (until - from) >= 72 hours, use custom url and data-timeout

```
url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=102400000&max_result_bytes=12800000&max_threads=2"
data-timeout = "30s"
query-params = [
{
duration = "72h",
url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=1024000000&max_result_bytes=128000000&max_threads=1",
data-timeout = "60s"
}
]
```

### Index table
See [index table](./index-table.md) documentation for details.

Expand Down Expand Up @@ -69,11 +86,12 @@ Only one tag used as filter for index field Tag1, see graphite_tagged table [str
So, if the first tag in filter is costly (poor selectivity), like environment (with several possible values), query perfomance will be degraded.
Tune this with `tagged-costs` options:

`
```
tagged-costs = {
"environment" = { cost: 100 },
"project" = { values-cost = { "HugeProject" = 90 } } # overwrite tag value cost for some value only
}`
}
```

Default cost is 0 and positive or negative numbers can be used. So if environment is first tag filter in query, it will used as primary only if no other filters with equal operation. Costs from values-cost also applied to regex match or wilrdcarded equal.

Expand Down
26 changes: 22 additions & 4 deletions doc/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,23 @@ Useless settings:
- `max_ast_elements`: the same
- `max_execution_time`: with `cancel_http_readonly_queries_on_client_close=1` and `data-timeout = "1m"` it's already covered.

### Query multi parameters (for overwrite default url and data-timeout)

For queries with duration (until - from) >= 72 hours, use custom url and data-timeout

```
url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=102400000&max_result_bytes=12800000&max_threads=2"
data-timeout = "30s"
query-params = [
{
duration = "72h",
url = "http://graphite:qwerty@localhost:8123/?readonly=2&log_queries=1&max_rows_to_read=1024000000&max_result_bytes=128000000&max_threads=1",
data-timeout = "60s"
}
]
```

### Index table
See [index table](./index-table.md) documentation for details.

Expand Down Expand Up @@ -72,11 +89,12 @@ Only one tag used as filter for index field Tag1, see graphite_tagged table [str
So, if the first tag in filter is costly (poor selectivity), like environment (with several possible values), query perfomance will be degraded.
Tune this with `tagged-costs` options:

`
```
tagged-costs = {
"environment" = { cost: 100 },
"project" = { values-cost = { "HugeProject" = 90 } } # overwrite tag value cost for some value only
}`
}
```

Default cost is 0 and positive or negative numbers can be used. So if environment is first tag filter in query, it will used as primary only if no other filters with equal operation. Costs from values-cost also applied to regex match or wilrdcarded equal.

Expand Down Expand Up @@ -112,9 +130,9 @@ It's possible to set multiple loggers. See `Config` description in [config.go](h
headers-to-log = []

[clickhouse]
# see https://clickhouse.tech/docs/en/interfaces/http
# default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params
url = "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1"
# total timeout to fetch data
# default total timeout to fetch data, can be overwritten with query-params
data-timeout = "1m0s"
# see doc/index-table.md
index-table = "graphite_index"
Expand Down
25 changes: 24 additions & 1 deletion render/data/multi_target.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"sync"
"time"

v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
"github.com/lomik/graphite-clickhouse/config"
Expand Down Expand Up @@ -58,6 +59,26 @@ func (m *MultiTarget) checkMetricsLimitExceeded(num int) error {
return nil
}

func getDataTimeout(cfg *config.Config, m *MultiTarget) time.Duration {
dataTimeout := cfg.ClickHouse.DataTimeout
if len(cfg.ClickHouse.QueryParams) > 1 {
var maxDuration time.Duration
for tf, _ := range *m {
duration := time.Second * time.Duration(tf.Until-tf.From)
if duration >= maxDuration {
maxDuration = duration
}
}

i := config.BinarySearchQueryParam(cfg.ClickHouse.QueryParams, maxDuration)
if i >= 0 {
return cfg.ClickHouse.QueryParams[i].DataTimeout
}
}

return dataTimeout
}

// Fetch fetches the parsed ClickHouse data returns CHResponses
func (m *MultiTarget) Fetch(ctx context.Context, cfg *config.Config, chContext string) (CHResponses, error) {
var lock sync.RWMutex
Expand All @@ -71,7 +92,9 @@ func (m *MultiTarget) Fetch(ctx context.Context, cfg *config.Config, chContext s
return nil, err
}

ctxTimeout, cancel := context.WithTimeout(ctx, cfg.ClickHouse.DataTimeout)
dataTimeout := getDataTimeout(cfg, m)

ctxTimeout, cancel := context.WithTimeout(ctx, dataTimeout)
defer cancel()

errors := make([]error, 0, len(*m))
Expand Down
154 changes: 154 additions & 0 deletions render/data/multi_target_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
package data

import (
"testing"
"time"

"github.com/lomik/graphite-clickhouse/config"
)

func Test_getDataTimeout(t *testing.T) {
tests := []struct {
name string
cfg *config.Config
m *MultiTarget
want time.Duration
}{
{
name: "one DataTimeout",
cfg: &config.Config{
ClickHouse: config.ClickHouse{
DataTimeout: time.Second,
QueryParams: []config.QueryParam{
{ // default params
Duration: 0,
DataTimeout: time.Second,
},
},
},
},
m: &MultiTarget{
TimeFrame{
From: 1647198000,
Until: 1647234000,
}: &Targets{},
},
want: time.Second,
},
{
name: "default DataTimeout",
cfg: &config.Config{
ClickHouse: config.ClickHouse{
DataTimeout: time.Second,
QueryParams: []config.QueryParam{
{ // default params
Duration: 0,
DataTimeout: time.Second,
},
{
Duration: time.Hour,
DataTimeout: time.Minute,
},
},
},
},
m: &MultiTarget{
TimeFrame{ // 1 hour - 1s
From: 1647198000,
Until: 1647201600 - 1,
}: &Targets{},
},
want: time.Second,
},
{
name: "1m DataTimeout (1 param), select 1h duration",
cfg: &config.Config{
ClickHouse: config.ClickHouse{
DataTimeout: time.Second * 10,
QueryParams: []config.QueryParam{
{ // default params
Duration: 0,
DataTimeout: time.Second,
},
{
Duration: time.Hour,
DataTimeout: time.Minute,
},
},
},
},
m: &MultiTarget{
TimeFrame{ // 1 hour
From: 1647198000,
Until: 1647201600,
}: &Targets{},
},
want: time.Minute,
},
{
name: "1m DataTimeout (2 param), select 1h duration",
cfg: &config.Config{
ClickHouse: config.ClickHouse{
DataTimeout: time.Second,
QueryParams: []config.QueryParam{
{ // default params
Duration: 0,
DataTimeout: time.Second,
},
{
Duration: time.Hour,
DataTimeout: time.Minute,
},
{
Duration: time.Hour * 2,
DataTimeout: 10 * time.Minute,
},
},
},
},
m: &MultiTarget{
TimeFrame{ // 1 hour
From: 1647198000,
Until: 1647201600,
}: &Targets{},
},
want: time.Minute,
},
{
name: "10m DataTimeout (2 param), select 2h1s duration",
cfg: &config.Config{
ClickHouse: config.ClickHouse{
DataTimeout: time.Second,
QueryParams: []config.QueryParam{
{ // default params
Duration: 0,
DataTimeout: time.Second,
},
{
Duration: time.Hour,
DataTimeout: time.Minute,
},
{
Duration: time.Hour * 2,
DataTimeout: 10 * time.Minute,
},
},
},
},
m: &MultiTarget{
TimeFrame{ // 2 hour 1s
From: 1647198000,
Until: 1647205201,
}: &Targets{},
},
want: 10 * time.Minute,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := getDataTimeout(tt.cfg, tt.m); got != tt.want {
t.Errorf("getDataTimeout() = %v, want %v", got, tt.want)
}
})
}
}
Loading

0 comments on commit c5000f1

Please sign in to comment.