From e41c6152b30aa039df05ccdcf2f3d7577d4941b4 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Wed, 10 Aug 2022 13:28:06 +0200 Subject: [PATCH 1/3] Track custom anonymous usage statistics from ingester and querier Signed-off-by: Marco Pracucci --- pkg/api/handlers.go | 31 +++++++---- pkg/ingester/ingester.go | 60 +++++++++++++++++++++ pkg/ingester/ingester_test.go | 22 ++++++++ pkg/mimir/modules.go | 3 ++ pkg/usagestats/middleware.go | 27 ++++++++++ pkg/usagestats/report.go | 28 +++++++++- pkg/usagestats/report_test.go | 10 ++++ pkg/usagestats/reporter.go | 6 ++- pkg/usagestats/stats.go | 98 +++++++++++++++++++++++++++++++++-- pkg/usagestats/stats_test.go | 25 +++++++++ 10 files changed, 292 insertions(+), 18 deletions(-) create mode 100644 pkg/usagestats/middleware.go create mode 100644 pkg/usagestats/stats_test.go diff --git a/pkg/api/handlers.go b/pkg/api/handlers.go index 0ed88d079b6..62dad7b668e 100644 --- a/pkg/api/handlers.go +++ b/pkg/api/handlers.go @@ -33,6 +33,7 @@ import ( "github.com/grafana/mimir/pkg/querier" "github.com/grafana/mimir/pkg/querier/stats" + "github.com/grafana/mimir/pkg/usagestats" "github.com/grafana/mimir/pkg/util" "github.com/grafana/mimir/pkg/util/validation" ) @@ -248,18 +249,28 @@ func NewQuerierHandler( promRouter := route.New().WithPrefix(path.Join(prefix, "/api/v1")) api.Register(promRouter) + // Track the requests count in the anonymous usage stats. + remoteReadStats := usagestats.NewRequestsMiddleware("querier_remote_read_requests") + instantQueryStats := usagestats.NewRequestsMiddleware("querier_instant_query_requests") + rangeQueryStats := usagestats.NewRequestsMiddleware("querier_range_query_requests") + exemplarsQueryStats := usagestats.NewRequestsMiddleware("querier_exemplars_query_requests") + labelsQueryStats := usagestats.NewRequestsMiddleware("querier_labels_query_requests") + seriesQueryStats := usagestats.NewRequestsMiddleware("querier_series_query_requests") + metadataQueryStats := usagestats.NewRequestsMiddleware("querier_metadata_query_requests") + cardinalityQueryStats := usagestats.NewRequestsMiddleware("querier_cardinality_query_requests") + // TODO(gotjosh): This custom handler is temporary until we're able to vendor the changes in: // https://github.com/prometheus/prometheus/pull/7125/files - router.Path(path.Join(prefix, "/api/v1/read")).Methods("POST").Handler(querier.RemoteReadHandler(queryable, logger)) - router.Path(path.Join(prefix, "/api/v1/query")).Methods("GET", "POST").Handler(promRouter) - router.Path(path.Join(prefix, "/api/v1/query_range")).Methods("GET", "POST").Handler(promRouter) - router.Path(path.Join(prefix, "/api/v1/query_exemplars")).Methods("GET", "POST").Handler(promRouter) - router.Path(path.Join(prefix, "/api/v1/labels")).Methods("GET", "POST").Handler(promRouter) - router.Path(path.Join(prefix, "/api/v1/label/{name}/values")).Methods("GET").Handler(promRouter) - router.Path(path.Join(prefix, "/api/v1/series")).Methods("GET", "POST", "DELETE").Handler(promRouter) - router.Path(path.Join(prefix, "/api/v1/metadata")).Methods("GET").Handler(querier.NewMetadataHandler(metadataSupplier)) - router.Path(path.Join(prefix, "/api/v1/cardinality/label_names")).Methods("GET", "POST").Handler(querier.LabelNamesCardinalityHandler(distributor, limits)) - router.Path(path.Join(prefix, "/api/v1/cardinality/label_values")).Methods("GET", "POST").Handler(querier.LabelValuesCardinalityHandler(distributor, limits)) + router.Path(path.Join(prefix, "/api/v1/read")).Methods("POST").Handler(remoteReadStats.Wrap(querier.RemoteReadHandler(queryable, logger))) + router.Path(path.Join(prefix, "/api/v1/query")).Methods("GET", "POST").Handler(instantQueryStats.Wrap(promRouter)) + router.Path(path.Join(prefix, "/api/v1/query_range")).Methods("GET", "POST").Handler(rangeQueryStats.Wrap(promRouter)) + router.Path(path.Join(prefix, "/api/v1/query_exemplars")).Methods("GET", "POST").Handler(exemplarsQueryStats.Wrap(promRouter)) + router.Path(path.Join(prefix, "/api/v1/labels")).Methods("GET", "POST").Handler(labelsQueryStats.Wrap(promRouter)) + router.Path(path.Join(prefix, "/api/v1/label/{name}/values")).Methods("GET").Handler(labelsQueryStats.Wrap(promRouter)) + router.Path(path.Join(prefix, "/api/v1/series")).Methods("GET", "POST", "DELETE").Handler(seriesQueryStats.Wrap(promRouter)) + router.Path(path.Join(prefix, "/api/v1/metadata")).Methods("GET").Handler(metadataQueryStats.Wrap(querier.NewMetadataHandler(metadataSupplier))) + router.Path(path.Join(prefix, "/api/v1/cardinality/label_names")).Methods("GET", "POST").Handler(cardinalityQueryStats.Wrap(querier.LabelNamesCardinalityHandler(distributor, limits))) + router.Path(path.Join(prefix, "/api/v1/cardinality/label_values")).Methods("GET", "POST").Handler(cardinalityQueryStats.Wrap(querier.LabelValuesCardinalityHandler(distributor, limits))) // Track execution time. return stats.NewWallTimeMiddleware().Wrap(router) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 1eca438c83b..f4f572001ce 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -10,6 +10,7 @@ package ingester import ( "context" + "expvar" "flag" "fmt" "io" @@ -55,6 +56,7 @@ import ( "github.com/grafana/mimir/pkg/storage/chunk" "github.com/grafana/mimir/pkg/storage/sharding" mimir_tsdb "github.com/grafana/mimir/pkg/storage/tsdb" + "github.com/grafana/mimir/pkg/usagestats" "github.com/grafana/mimir/pkg/util" "github.com/grafana/mimir/pkg/util/globalerror" util_log "github.com/grafana/mimir/pkg/util/log" @@ -74,6 +76,9 @@ const ( // Period at which to attempt purging metadata from memory. metadataPurgePeriod = 5 * time.Minute + // How frequently update the usage statistics. + usageStatsUpdateInterval = usagestats.DefaultReportSendInterval / 10 + // IngesterRingKey is the key under which we store the ingesters ring in the KVStore. IngesterRingKey = "ring" @@ -88,6 +93,12 @@ const ( sampleTooOld = "sample-too-old" newValueForTimestamp = "new-value-for-timestamp" sampleOutOfBounds = "sample-out-of-bounds" + + replicationFactorStatsName = "ingester_replication_factor" + memorySeriesStatsName = "ingester_inmemory_series" + memoryTenantsStatsName = "ingester_inmemory_tenants" + appendedSamplesStatsName = "ingester_appended_samples" + appendedExemplarsStatsName = "ingester_appended_exemplars" ) // BlocksUploader interface is used to have an easy way to mock it in tests. @@ -225,6 +236,12 @@ type Ingester struct { // Rate of pushed samples. Used to limit global samples push rate. ingestionRate *util_math.EwmaRate inflightPushRequests atomic.Int64 + + // Anonymous usage statistics tracked by ingester. + memorySeriesStats *expvar.Int + memoryTenantsStats *expvar.Int + appendedSamplesStats *usagestats.Counter + appendedExemplarsStats *usagestats.Counter } func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { @@ -237,6 +254,9 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus return nil, errors.Wrap(err, "failed to create the bucket client") } + // Track constant usage stats. + usagestats.GetInt(replicationFactorStatsName).Set(int64(cfg.IngesterRing.ReplicationFactor)) + return &Ingester{ cfg: cfg, limits: limits, @@ -249,6 +269,11 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus forceCompactTrigger: make(chan requestWithUsersAndCallback), shipTrigger: make(chan requestWithUsersAndCallback), seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), + + memorySeriesStats: usagestats.GetAndResetInt(memorySeriesStatsName), + memoryTenantsStats: usagestats.GetAndResetInt(memoryTenantsStatsName), + appendedSamplesStats: usagestats.GetAndResetCounter(appendedSamplesStatsName), + appendedExemplarsStats: usagestats.GetAndResetCounter(appendedExemplarsStatsName), }, nil } @@ -427,6 +452,9 @@ func (i *Ingester) updateLoop(ctx context.Context) error { metadataPurgeTicker := time.NewTicker(metadataPurgePeriod) defer metadataPurgeTicker.Stop() + usageStatsUpdateTicker := time.NewTicker(util.DurationWithJitter(usageStatsUpdateInterval, 0.2)) + defer usageStatsUpdateTicker.Stop() + for { select { case <-metadataPurgeTicker.C: @@ -447,6 +475,9 @@ func (i *Ingester) updateLoop(ctx context.Context) error { case <-activeSeriesTickerChan: i.updateActiveSeries(time.Now()) + case <-usageStatsUpdateTicker.C: + i.updateUsageStats() + case <-ctx.Done(): return nil case err := <-i.subservicesWatcher.Chan(): @@ -495,6 +526,30 @@ func (i *Ingester) updateActiveSeries(now time.Time) { } } +// updateUsageStats updated some anonymous usage statistics tracked by the ingester. +// This function is expected to be called periodically. +func (i *Ingester) updateUsageStats() { + memoryUsersCount := int64(0) + memorySeriesCount := int64(0) + + for _, userID := range i.getTSDBUsers() { + userDB := i.getTSDB(userID) + if userDB == nil { + continue + } + + // Count only tenants with at least 1 series. + if numSeries := userDB.Head().NumSeries(); numSeries > 0 { + memoryUsersCount++ + memorySeriesCount += int64(numSeries) + } + } + + // Track anonymous usage stats. + i.memorySeriesStats.Set(memorySeriesCount) + i.memoryTenantsStats.Set(memoryUsersCount) +} + // applyTSDBSettings goes through all tenants and applies // * The current max-exemplars setting. If it changed, tsdb will resize the buffer; if it didn't change tsdb will return quickly. // * The current out-of-order time window. If it changes from 0 to >0, then a new Write-Behind-Log gets created for that tenant. @@ -797,6 +852,8 @@ func (i *Ingester) PushWithCleanup(ctx context.Context, req *mimirpb.WriteReques i.metrics.ingestedSamplesFail.WithLabelValues(userID).Add(float64(failedSamplesCount)) i.metrics.ingestedExemplars.Add(float64(succeededExemplarsCount)) i.metrics.ingestedExemplarsFail.Add(float64(failedExemplarsCount)) + i.appendedSamplesStats.Inc(int64(succeededSamplesCount)) + i.appendedExemplarsStats.Inc(int64(succeededExemplarsCount)) if sampleOutOfBoundsCount > 0 { validation.DiscardedSamples.WithLabelValues(sampleOutOfBounds, userID).Add(float64(sampleOutOfBoundsCount)) @@ -1693,6 +1750,9 @@ func (i *Ingester) openExistingTSDB(ctx context.Context) error { return err } + // Update the usage statistics once all TSDBs have been opened. + i.updateUsageStats() + level.Info(i.logger).Log("msg", "successfully opened existing TSDBs") return nil } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 60a8e122872..f588e6854a7 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -58,6 +58,7 @@ import ( "github.com/grafana/mimir/pkg/storage/chunk" "github.com/grafana/mimir/pkg/storage/sharding" mimir_tsdb "github.com/grafana/mimir/pkg/storage/tsdb" + "github.com/grafana/mimir/pkg/usagestats" "github.com/grafana/mimir/pkg/util" "github.com/grafana/mimir/pkg/util/chunkcompat" util_math "github.com/grafana/mimir/pkg/util/math" @@ -801,6 +802,27 @@ func TestIngester_Push(t *testing.T) { // Check tracked Prometheus metrics err = testutil.GatherAndCompare(registry, strings.NewReader(testData.expectedMetrics), mn...) assert.NoError(t, err) + + // Check anonymous usage stats. + expectedTenantsCount := 0 + expectedSamplesCount := 0 + expectedExemplarsCount := 0 + if len(testData.expectedIngested) > 0 { + expectedTenantsCount = 1 + } + for _, stream := range testData.expectedIngested { + expectedSamplesCount += len(stream.Values) + } + for _, series := range testData.expectedExemplarsIngested { + expectedExemplarsCount += len(series.Exemplars) + } + + i.updateUsageStats() + + assert.Equal(t, int64(len(testData.expectedIngested)), usagestats.GetInt(memorySeriesStatsName).Value()) + assert.Equal(t, int64(expectedTenantsCount), usagestats.GetInt(memoryTenantsStatsName).Value()) + assert.Equal(t, int64(expectedSamplesCount), usagestats.GetCounter(appendedSamplesStatsName).Total()) + assert.Equal(t, int64(expectedExemplarsCount), usagestats.GetCounter(appendedExemplarsStatsName).Total()) }) } } diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 75629ebbb99..3e425a22f55 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -760,6 +760,9 @@ func (t *Mimir) initUsageStats() (services.Service, error) { return nil, err } + // Track anonymous usage statistics. + usagestats.GetString("blocks_storage_backend").Set(t.Cfg.BlocksStorage.Bucket.Backend) + t.UsageStatsReporter = usagestats.NewReporter(bucketClient, util_log.Logger, prometheus.DefaultRegisterer) return t.UsageStatsReporter, nil } diff --git a/pkg/usagestats/middleware.go b/pkg/usagestats/middleware.go new file mode 100644 index 00000000000..6717d679c4c --- /dev/null +++ b/pkg/usagestats/middleware.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package usagestats + +import ( + "net/http" +) + +// RequestsMiddleware tracks the number of requests. +type RequestsMiddleware struct { + counter *Counter +} + +// NewRequestsMiddleware makes a new RequestsMiddleware. +func NewRequestsMiddleware(name string) *RequestsMiddleware { + return &RequestsMiddleware{ + counter: GetCounter(name), + } +} + +// Wrap implements middleware.Interface. +func (m *RequestsMiddleware) Wrap(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m.counter.Inc(1) + next.ServeHTTP(w, r) + }) +} diff --git a/pkg/usagestats/report.go b/pkg/usagestats/report.go index 3d695247dc9..6ec89269004 100644 --- a/pkg/usagestats/report.go +++ b/pkg/usagestats/report.go @@ -5,6 +5,7 @@ package usagestats import ( "expvar" "runtime" + "strings" "time" prom "github.com/prometheus/prometheus/web/api/v1" @@ -78,11 +79,36 @@ func buildReport(seed ClusterSeed, reportAt time.Time, reportInterval time.Durat // buildMetrics builds the metrics part of the report to be sent to the stats server. func buildMetrics() map[string]interface{} { - return map[string]interface{}{ + result := map[string]interface{}{ "memstats": buildMemstats(), "num_cpu": runtime.NumCPU(), "num_goroutine": runtime.NumGoroutine(), } + + expvar.Do(func(kv expvar.KeyValue) { + if !strings.HasPrefix(kv.Key, statsPrefix) || kv.Key == statsPrefix+targetKey || kv.Key == statsPrefix+editionKey { + return + } + + var value interface{} + switch v := kv.Value.(type) { + case *expvar.Int: + value = v.Value() + case *expvar.String: + value = v.Value() + case *Counter: + v.updateRate() + value = v.Value() + v.reset() + default: + // Unsupported. + return + } + + result[strings.TrimPrefix(kv.Key, statsPrefix)] = value + }) + + return result } func buildMemstats() interface{} { diff --git a/pkg/usagestats/report_test.go b/pkg/usagestats/report_test.go index a3c9ecffcfb..b6916ff1b28 100644 --- a/pkg/usagestats/report_test.go +++ b/pkg/usagestats/report_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/grafana/mimir/pkg/util/version" ) @@ -25,6 +26,10 @@ func TestBuildReport(t *testing.T) { version.Branch = "dev-branch" version.Revision = "dev-revision" + GetString("custom_string").Set("my_value") + GetInt("custom_int").Set(111) + GetCounter("custom_counter").Inc(222) + report := buildReport(seed, reportAt, reportInterval) assert.Equal(t, "test", report.ClusterID) assert.Equal(t, clusterCreatedAt, report.CreatedAt) @@ -39,4 +44,9 @@ func TestBuildReport(t *testing.T) { assert.Equal(t, "dev-branch", report.Version.Branch) assert.Equal(t, "dev-revision", report.Version.Revision) assert.Equal(t, runtime.Version(), report.Version.GoVersion) + assert.Equal(t, "my_value", report.Metrics["custom_string"]) + assert.Equal(t, int64(111), report.Metrics["custom_int"]) + + require.IsType(t, map[string]interface{}{}, report.Metrics["custom_counter"]) + assert.Equal(t, int64(222), report.Metrics["custom_counter"].(map[string]interface{})["total"]) } diff --git a/pkg/usagestats/reporter.go b/pkg/usagestats/reporter.go index 93d8210dea2..a2db1d7e7a8 100644 --- a/pkg/usagestats/reporter.go +++ b/pkg/usagestats/reporter.go @@ -26,8 +26,10 @@ import ( ) const ( + // DefaultReportSendInterval is the interval at which anonymous usage statistics are reported. + DefaultReportSendInterval = 4 * time.Hour + defaultReportCheckInterval = time.Minute - defaultReportSendInterval = 4 * time.Hour defaultStatsServerURL = "https://stats.grafana.org/mimir-usage-report" ) @@ -74,7 +76,7 @@ func NewReporter(bucketClient objstore.InstrumentedBucket, logger log.Logger, re client: http.Client{Timeout: 5 * time.Second}, serverURL: defaultStatsServerURL, reportCheckInterval: defaultReportCheckInterval, - reportSendInterval: defaultReportSendInterval, + reportSendInterval: DefaultReportSendInterval, seedFileMinStability: clusterSeedFileMinStability, requestsTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{ diff --git a/pkg/usagestats/stats.go b/pkg/usagestats/stats.go index a74bb90f2cc..1a575548894 100644 --- a/pkg/usagestats/stats.go +++ b/pkg/usagestats/stats.go @@ -3,8 +3,12 @@ package usagestats import ( + "encoding/json" "expvar" "fmt" + "time" + + "go.uber.org/atomic" ) const ( @@ -22,17 +26,17 @@ func init() { // SetTarget sets the target name. func SetTarget(target string) { - getString(targetKey).Set(target) + GetString(targetKey).Set(target) } // SetEdition sets the edition name. func SetEdition(edition string) { - getString(editionKey).Set(edition) + GetString(editionKey).Set(edition) } -// getString returns the String stats object for the given name. -// It creates the stats object if doesn't exist yet. -func getString(name string) *expvar.String { +// GetString returns the String stats object for the given name. +// It creates the stats object if it doesn't exist yet. +func GetString(name string) *expvar.String { existing := expvar.Get(statsPrefix + name) if existing != nil { if s, ok := existing.(*expvar.String); ok { @@ -42,3 +46,87 @@ func getString(name string) *expvar.String { } return expvar.NewString(statsPrefix + name) } + +// GetInt returns a new Int stats object for the given name. +// It creates the stats object if it doesn't exist yet. +func GetInt(name string) *expvar.Int { + existing := expvar.Get(statsPrefix + name) + if existing != nil { + if i, ok := existing.(*expvar.Int); ok { + return i + } + panic(fmt.Sprintf("%v is set to a non-int value", name)) + } + return expvar.NewInt(statsPrefix + name) +} + +// GetAndResetInt calls GetInt and then reset it to 0. +func GetAndResetInt(name string) *expvar.Int { + stat := GetInt(name) + stat.Set(0) + return stat +} + +type Counter struct { + total *atomic.Int64 + rate *atomic.Float64 + + resetTime time.Time +} + +// GetCounter returns a new Counter stats object for the given name. +// It creates the stats object if it doesn't exist yet. +func GetCounter(name string) *Counter { + c := &Counter{ + total: atomic.NewInt64(0), + rate: atomic.NewFloat64(0), + resetTime: time.Now(), + } + existing := expvar.Get(statsPrefix + name) + if existing != nil { + if c, ok := existing.(*Counter); ok { + return c + } + panic(fmt.Sprintf("%v is set to a non-Counter value", name)) + } + expvar.Publish(statsPrefix+name, c) + return c +} + +// GetAndResetCounter calls GetCounter and then reset it to 0. +func GetAndResetCounter(name string) *Counter { + stat := GetCounter(name) + stat.reset() + return stat +} + +func (c *Counter) updateRate() { + total := c.total.Load() + c.rate.Store(float64(total) / time.Since(c.resetTime).Seconds()) +} + +func (c *Counter) reset() { + c.total.Store(0) + c.rate.Store(0) + c.resetTime = time.Now() +} + +func (c *Counter) Inc(i int64) { + c.total.Add(i) +} + +func (c *Counter) String() string { + b, _ := json.Marshal(c.Value()) + return string(b) +} + +func (c *Counter) Total() int64 { + return c.total.Load() +} + +func (c *Counter) Value() map[string]interface{} { + return map[string]interface{}{ + "total": c.total.Load(), + "rate": c.rate.Load(), + } +} diff --git a/pkg/usagestats/stats_test.go b/pkg/usagestats/stats_test.go new file mode 100644 index 00000000000..da5a670c2fa --- /dev/null +++ b/pkg/usagestats/stats_test.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package usagestats + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCounter(t *testing.T) { + c := GetCounter("test_counter") + c.Inc(100) + c.Inc(200) + c.Inc(300) + time.Sleep(1 * time.Second) + c.updateRate() + v := c.Value() + require.Equal(t, int64(600), v["total"]) + require.GreaterOrEqual(t, v["rate"], float64(590)) + c.reset() + require.Equal(t, int64(0), c.Value()["total"]) + require.Equal(t, float64(0), c.Value()["rate"]) +} From 710ba310baf517b69aa1f3b7a599d8339e6357fc Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Wed, 10 Aug 2022 13:59:00 +0200 Subject: [PATCH 2/3] Fixed race condition Signed-off-by: Marco Pracucci --- pkg/usagestats/stats.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/usagestats/stats.go b/pkg/usagestats/stats.go index 1a575548894..423f6d4af4f 100644 --- a/pkg/usagestats/stats.go +++ b/pkg/usagestats/stats.go @@ -68,10 +68,9 @@ func GetAndResetInt(name string) *expvar.Int { } type Counter struct { - total *atomic.Int64 - rate *atomic.Float64 - - resetTime time.Time + total *atomic.Int64 + rate *atomic.Float64 + resetTime *atomic.Time } // GetCounter returns a new Counter stats object for the given name. @@ -80,7 +79,7 @@ func GetCounter(name string) *Counter { c := &Counter{ total: atomic.NewInt64(0), rate: atomic.NewFloat64(0), - resetTime: time.Now(), + resetTime: atomic.NewTime(time.Now()), } existing := expvar.Get(statsPrefix + name) if existing != nil { @@ -102,13 +101,13 @@ func GetAndResetCounter(name string) *Counter { func (c *Counter) updateRate() { total := c.total.Load() - c.rate.Store(float64(total) / time.Since(c.resetTime).Seconds()) + c.rate.Store(float64(total) / time.Since(c.resetTime.Load()).Seconds()) } func (c *Counter) reset() { c.total.Store(0) c.rate.Store(0) - c.resetTime = time.Now() + c.resetTime.Store(time.Now()) } func (c *Counter) Inc(i int64) { From ddbf2deee62545379892ef67ee063cae0de49ec7 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Wed, 10 Aug 2022 18:01:41 +0200 Subject: [PATCH 3/3] Removed jitter because not useful Signed-off-by: Marco Pracucci --- pkg/ingester/ingester.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index f4f572001ce..5d98d2fa617 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -452,7 +452,7 @@ func (i *Ingester) updateLoop(ctx context.Context) error { metadataPurgeTicker := time.NewTicker(metadataPurgePeriod) defer metadataPurgeTicker.Stop() - usageStatsUpdateTicker := time.NewTicker(util.DurationWithJitter(usageStatsUpdateInterval, 0.2)) + usageStatsUpdateTicker := time.NewTicker(usageStatsUpdateInterval) defer usageStatsUpdateTicker.Stop() for {