From 04566a535ee0ec5d83059ff6b0620233f85c0be3 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 4 May 2023 01:51:56 -0700 Subject: [PATCH] Disable active query tracker in querier test (#5318) * remove parallel tests if we create a query engine Signed-off-by: Ben Ye * disable active query tracker Signed-off-by: Ben Ye --------- Signed-off-by: Ben Ye --- go.mod | 2 +- go.sum | 4 +- .../blocks_consistency_checker_test.go | 2 - pkg/querier/blocks_store_queryable_test.go | 1 + pkg/querier/chunk_store_queryable_test.go | 8 +-- pkg/querier/error_translate_queryable_test.go | 2 - pkg/querier/querier_test.go | 50 +++++++++++-------- .../promql-engine/engine/engine.go | 15 +++++- .../promql-engine/execution/binary/vector.go | 4 ++ .../execution/scan/matrix_selector.go | 32 ++---------- vendor/modules.txt | 2 +- 11 files changed, 58 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index 85965e6d90..9396797de7 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/sony/gobreaker v0.5.0 github.com/spf13/afero v1.9.5 github.com/stretchr/testify v1.8.2 - github.com/thanos-community/promql-engine v0.0.0-20230424192336-6fec99413de8 + github.com/thanos-community/promql-engine v0.0.0-20230503192559-c9c196cc08b3 github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204 github.com/thanos-io/thanos v0.31.1-0.20230418080134-0c2af2155056 github.com/uber/jaeger-client-go v2.30.0+incompatible diff --git a/go.sum b/go.sum index 12ff1500fe..a5bb159952 100644 --- a/go.sum +++ b/go.sum @@ -1864,8 +1864,8 @@ github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= -github.com/thanos-community/promql-engine v0.0.0-20230424192336-6fec99413de8 h1:BQZw3InOmxnOb27fpdG/5gWEOydSQu+ylCsxGLRtMaY= -github.com/thanos-community/promql-engine v0.0.0-20230424192336-6fec99413de8/go.mod h1:PbimG7ocz5JmFRLlQ7yMcewnkunNBmvMyVgAoNmyvDw= +github.com/thanos-community/promql-engine v0.0.0-20230503192559-c9c196cc08b3 h1:WkhJnkQIHooPQ3A9vRvb0yub8+q3pTNshLvg0rU8YiA= +github.com/thanos-community/promql-engine v0.0.0-20230503192559-c9c196cc08b3/go.mod h1:PbimG7ocz5JmFRLlQ7yMcewnkunNBmvMyVgAoNmyvDw= github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204 h1:W4w5Iph7j32Sf1QFWLJDCqvO0WgZS0jHGID+qnq3wV0= github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204/go.mod h1:STSgpY8M6EKF2G/raUFdbIMf2U9GgYlEjAEHJxjvpAo= github.com/thanos-io/thanos v0.31.1-0.20230418080134-0c2af2155056 h1:x0jPHfsMnf8DtdxD4B2nMZAbIXza3y/wWgmVLV+3Gxc= diff --git a/pkg/querier/blocks_consistency_checker_test.go b/pkg/querier/blocks_consistency_checker_test.go index a5d4b4cab7..83bf0d70fb 100644 --- a/pkg/querier/blocks_consistency_checker_test.go +++ b/pkg/querier/blocks_consistency_checker_test.go @@ -15,8 +15,6 @@ import ( ) func TestBlocksConsistencyChecker_Check(t *testing.T) { - //parallel testing causes data race - now := time.Now() uploadGracePeriod := 10 * time.Minute deletionGracePeriod := 5 * time.Minute diff --git a/pkg/querier/blocks_store_queryable_test.go b/pkg/querier/blocks_store_queryable_test.go index 2e88d0c302..e57b010304 100644 --- a/pkg/querier/blocks_store_queryable_test.go +++ b/pkg/querier/blocks_store_queryable_test.go @@ -1336,6 +1336,7 @@ func TestBlocksStoreQuerier_SelectSortedShouldHonorQueryStoreAfter(t *testing.T) } func TestBlocksStoreQuerier_PromQLExecution(t *testing.T) { + t.Parallel() logger := log.NewNopLogger() opts := promql.EngineOpts{ Logger: logger, diff --git a/pkg/querier/chunk_store_queryable_test.go b/pkg/querier/chunk_store_queryable_test.go index 3b3f28f575..f216532523 100644 --- a/pkg/querier/chunk_store_queryable_test.go +++ b/pkg/querier/chunk_store_queryable_test.go @@ -24,11 +24,11 @@ import ( var _ SeriesWithChunks = &chunkSeries{} func TestChunkQueryable(t *testing.T) { + t.Parallel() opts := promql.EngineOpts{ - Logger: log.NewNopLogger(), - ActiveQueryTracker: promql.NewActiveQueryTracker(t.TempDir(), 10, log.NewNopLogger()), - MaxSamples: 1e6, - Timeout: 1 * time.Minute, + Logger: log.NewNopLogger(), + MaxSamples: 1e6, + Timeout: 1 * time.Minute, } for _, thanosEngine := range []bool{false, true} { for _, testcase := range testcases { diff --git a/pkg/querier/error_translate_queryable_test.go b/pkg/querier/error_translate_queryable_test.go index d79d6be782..d5d93b72a5 100644 --- a/pkg/querier/error_translate_queryable_test.go +++ b/pkg/querier/error_translate_queryable_test.go @@ -26,8 +26,6 @@ import ( ) func TestApiStatusCodes(t *testing.T) { - t.Parallel() - for ix, tc := range []struct { err error expectedString string diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index e048252b09..5f3d37ea2d 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -156,6 +156,7 @@ var ( ) func TestShouldSortSeriesIfQueryingMultipleQueryables(t *testing.T) { + t.Parallel() start := time.Now().Add(-2 * time.Hour) end := time.Now() ctx := user.InjectOrgID(context.Background(), "0") @@ -274,13 +275,10 @@ func TestShouldSortSeriesIfQueryingMultipleQueryables(t *testing.T) { wQueriables = append(wQueriables, &wrappedSampleAndChunkQueryable{QueryableWithFilter: queriable}) } queryable := NewQueryable(wDistributorQueriable, wQueriables, batch.NewChunkMergeIterator, cfg, overrides, purger.NewNoopTombstonesLoader()) - queryTracker := promql.NewActiveQueryTracker(t.TempDir(), 10, log.NewNopLogger()) - opts := promql.EngineOpts{ - Logger: log.NewNopLogger(), - ActiveQueryTracker: queryTracker, - MaxSamples: 1e6, - Timeout: 1 * time.Minute, + Logger: log.NewNopLogger(), + MaxSamples: 1e6, + Timeout: 1 * time.Minute, } var queryEngine v1.QueryEngine if thanosEngine { @@ -310,6 +308,7 @@ func TestShouldSortSeriesIfQueryingMultipleQueryables(t *testing.T) { } func TestQuerier(t *testing.T) { + t.Parallel() var cfg Config flagext.DefaultValues(&cfg) @@ -322,10 +321,9 @@ func TestQuerier(t *testing.T) { db, _ := mockTSDB(t, []labels.Labels{lset}, model.Time(0), int(chunks*samplesPerChunk), sampleRate, chunkOffset, int(samplesPerChunk)) opts := promql.EngineOpts{ - Logger: log.NewNopLogger(), - ActiveQueryTracker: promql.NewActiveQueryTracker(t.TempDir(), 10, log.NewNopLogger()), - MaxSamples: 1e6, - Timeout: 1 * time.Minute, + Logger: log.NewNopLogger(), + MaxSamples: 1e6, + Timeout: 1 * time.Minute, } for _, thanosEngine := range []bool{false, true} { for _, query := range queries { @@ -345,6 +343,8 @@ func TestQuerier(t *testing.T) { } cfg.IngesterStreaming = streaming cfg.Iterators = iterators + // Disable active query tracker to avoid mmap error. + cfg.ActiveQueryTrackerDir = "" chunkStore, through := makeMockChunkStore(t, chunks, encoding.e) distributor := mockDistibutorFor(t, chunkStore, through) @@ -405,6 +405,7 @@ func mockTSDB(t *testing.T, labels []labels.Labels, mint model.Time, samples int } func TestNoHistoricalQueryToIngester(t *testing.T) { + t.Parallel() testCases := []struct { name string mint, maxt time.Time @@ -448,14 +449,14 @@ func TestNoHistoricalQueryToIngester(t *testing.T) { }, } - queryTracker := promql.NewActiveQueryTracker(t.TempDir(), 10, log.NewNopLogger()) opts := promql.EngineOpts{ - Logger: log.NewNopLogger(), - ActiveQueryTracker: queryTracker, - MaxSamples: 1e6, - Timeout: 1 * time.Minute, + Logger: log.NewNopLogger(), + MaxSamples: 1e6, + Timeout: 1 * time.Minute, } cfg := Config{} + // Disable active query tracker to avoid mmap error. + cfg.ActiveQueryTrackerDir = "" for _, ingesterStreaming := range []bool{true, false} { for _, thanosEngine := range []bool{true, false} { cfg.IngesterStreaming = ingesterStreaming @@ -557,6 +558,8 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryIntoFuture(t *testing.T) { cfg := Config{} flagext.DefaultValues(&cfg) + // Disable active query tracker to avoid mmap error. + cfg.ActiveQueryTrackerDir = "" for _, ingesterStreaming := range []bool{true, false} { cfg.IngesterStreaming = ingesterStreaming @@ -648,6 +651,8 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLength(t *testing.T) { //parallel testing causes data race var cfg Config flagext.DefaultValues(&cfg) + // Disable active query tracker to avoid mmap error. + cfg.ActiveQueryTrackerDir = "" limits := DefaultLimitsConfig() limits.MaxQueryLength = model.Duration(maxQueryLength) @@ -785,6 +790,8 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { flagext.DefaultValues(&cfg) cfg.IngesterStreaming = ingesterStreaming cfg.IngesterMetadataStreaming = ingesterStreaming + // Disable active query tracker to avoid mmap error. + cfg.ActiveQueryTrackerDir = "" limits := DefaultLimitsConfig() limits.MaxQueryLookback = testData.maxQueryLookback @@ -1219,7 +1226,7 @@ func (q *mockStoreQuerier) Close() error { } func TestShortTermQueryToLTS(t *testing.T) { - //parallel testing causes data race + t.Parallel() testCases := []struct { name string mint, maxt time.Time @@ -1257,17 +1264,16 @@ func TestShortTermQueryToLTS(t *testing.T) { }, } - queryTracker := promql.NewActiveQueryTracker(t.TempDir(), 10, log.NewNopLogger()) - engine := promql.NewEngine(promql.EngineOpts{ - Logger: log.NewNopLogger(), - ActiveQueryTracker: queryTracker, - MaxSamples: 1e6, - Timeout: 1 * time.Minute, + Logger: log.NewNopLogger(), + MaxSamples: 1e6, + Timeout: 1 * time.Minute, }) cfg := Config{} flagext.DefaultValues(&cfg) + // Disable active query tracker to avoid mmap error. + cfg.ActiveQueryTrackerDir = "" for _, ingesterStreaming := range []bool{true, false} { cfg.IngesterStreaming = ingesterStreaming diff --git a/vendor/github.com/thanos-community/promql-engine/engine/engine.go b/vendor/github.com/thanos-community/promql-engine/engine/engine.go index 21b635c556..f50142a465 100644 --- a/vendor/github.com/thanos-community/promql-engine/engine/engine.go +++ b/vendor/github.com/thanos-community/promql-engine/engine/engine.go @@ -76,9 +76,11 @@ type Opts struct { func (o Opts) getLogicalOptimizers() []logicalplan.Optimizer { var optimizers []logicalplan.Optimizer if o.LogicalOptimizers == nil { - optimizers = logicalplan.DefaultOptimizers + optimizers = make([]logicalplan.Optimizer, len(logicalplan.DefaultOptimizers)) + copy(optimizers, logicalplan.DefaultOptimizers) } else { - optimizers = o.LogicalOptimizers + optimizers = make([]logicalplan.Optimizer, len(o.LogicalOptimizers)) + copy(optimizers, o.LogicalOptimizers) } return append(optimizers, logicalplan.TrimSortFunctions{}) } @@ -331,6 +333,15 @@ func (e *compatibilityEngine) NewRangeQuery(ctx context.Context, q storage.Query }, nil } +type ExplainableQuery interface { + promql.Query + + Explain() string + Profile() +} + +var _ ExplainableQuery = &compatibilityQuery{} + type Query struct { exec model.VectorOperator opts *promql.QueryOpts diff --git a/vendor/github.com/thanos-community/promql-engine/execution/binary/vector.go b/vendor/github.com/thanos-community/promql-engine/execution/binary/vector.go index 8f2af47648..18251851fb 100644 --- a/vendor/github.com/thanos-community/promql-engine/execution/binary/vector.go +++ b/vendor/github.com/thanos-community/promql-engine/execution/binary/vector.go @@ -6,6 +6,7 @@ package binary import ( "context" "fmt" + "sort" "sync" "github.com/efficientgo/core/errors" @@ -349,6 +350,9 @@ func buildOutputSeries(seriesID uint64, highCardSeries, lowCardSeries model.Seri Keep(includeLabels...). Labels() metric = append(metric, lowCardLabels...) + sort.Slice(metric, func(i, j int) bool { + return metric[i].Name < metric[j].Name + }) } return model.Series{ID: seriesID, Metric: metric} } diff --git a/vendor/github.com/thanos-community/promql-engine/execution/scan/matrix_selector.go b/vendor/github.com/thanos-community/promql-engine/execution/scan/matrix_selector.go index b95651e93a..a829f02289 100644 --- a/vendor/github.com/thanos-community/promql-engine/execution/scan/matrix_selector.go +++ b/vendor/github.com/thanos-community/promql-engine/execution/scan/matrix_selector.go @@ -275,15 +275,7 @@ loop: switch buf.Next() { case chunkenc.ValNone: break loop - case chunkenc.ValHistogram: - t, h := buf.AtHistogram() - if value.IsStaleNaN(h.Sum) { - continue loop - } - if t >= mint { - out = append(out, promql.Sample{T: t, H: h.ToFloat()}) - } - case chunkenc.ValFloatHistogram: + case chunkenc.ValHistogram, chunkenc.ValFloatHistogram: t, fh := buf.AtFloatHistogram() if value.IsStaleNaN(fh.Sum) { continue loop @@ -305,13 +297,7 @@ loop: // The sought sample might also be in the range. switch soughtValueType { - case chunkenc.ValHistogram: - t, h := it.AtHistogram() - if t == maxt && !value.IsStaleNaN(h.Sum) { - out = append(out, promql.Sample{T: t, H: h.ToFloat()}) - } - - case chunkenc.ValFloatHistogram: + case chunkenc.ValHistogram, chunkenc.ValFloatHistogram: t, fh := it.AtFloatHistogram() if t == maxt && !value.IsStaleNaN(fh.Sum) { out = append(out, promql.Sample{T: t, H: fh}) @@ -379,12 +365,7 @@ loop: switch buf.Next() { case chunkenc.ValNone: break loop - case chunkenc.ValHistogram: - t, h := buf.AtHistogram() - if t >= mint { - out = append(out, promql.Sample{T: t, H: h.ToFloat()}) - } - case chunkenc.ValFloatHistogram: + case chunkenc.ValHistogram, chunkenc.ValFloatHistogram: t, fh := buf.AtFloatHistogram() if value.IsStaleNaN(fh.Sum) { continue loop @@ -413,12 +394,7 @@ loop: // The sought sample might also be in the range. switch soughtValueType { - case chunkenc.ValHistogram: - t, h := it.AtHistogram() - if t == maxt { - out = append(out, promql.Sample{T: t, H: h.ToFloat()}) - } - case chunkenc.ValFloatHistogram: + case chunkenc.ValHistogram, chunkenc.ValFloatHistogram: t, fh := it.AtFloatHistogram() if t == maxt && !value.IsStaleNaN(fh.Sum) { out = append(out, promql.Sample{T: t, H: fh}) diff --git a/vendor/modules.txt b/vendor/modules.txt index 037b8da61c..739adb72b1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -815,7 +815,7 @@ github.com/stretchr/objx github.com/stretchr/testify/assert github.com/stretchr/testify/mock github.com/stretchr/testify/require -# github.com/thanos-community/promql-engine v0.0.0-20230424192336-6fec99413de8 +# github.com/thanos-community/promql-engine v0.0.0-20230503192559-c9c196cc08b3 ## explicit; go 1.19 github.com/thanos-community/promql-engine/api github.com/thanos-community/promql-engine/engine