From 33bd13e3a6142c4a355358e410dd44c51e5dae1a Mon Sep 17 00:00:00 2001 From: Travis Patterson Date: Mon, 13 Jun 2022 09:56:51 -0600 Subject: [PATCH] Don't interupt the query path if deletes aren't available (#6368) (cherry picked from commit efda5b2af7bf67a0eda76c1fff7c86862e0b24fb) --- pkg/loki/modules.go | 2 +- pkg/querier/querier.go | 2 +- .../deletion/delete_requests_client.go | 8 ++++++- .../deletion/delete_requests_client_test.go | 4 ++-- .../shipper/compactor/deletion/metrics.go | 23 +++++++++++++++++++ 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 3e2e84ab8b237..32f1185bd4ec7 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -980,7 +980,7 @@ func (t *Loki) deleteRequestsClient() (deletion.DeleteRequestsClient, error) { return nil, err } - return deletion.NewDeleteRequestsClient(compactorAddress, &http.Client{Timeout: 5 * time.Second}) + return deletion.NewDeleteRequestsClient(compactorAddress, &http.Client{Timeout: 5 * time.Second}, prometheus.DefaultRegisterer) } func calculateMaxLookBack(pc config.PeriodConfig, maxLookBackConfig, minDuration time.Duration) (time.Duration, error) { diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 7bf9b1ca6f653..0fb699a67319b 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -175,7 +175,7 @@ func (q *SingleTenantQuerier) SelectSamples(ctx context.Context, params logql.Se params.SampleQueryRequest.Deletes, err = q.deletesForUser(ctx, params.Start, params.End) if err != nil { - return nil, err + level.Error(spanlogger.FromContext(ctx)).Log("msg", "failed loading deletes for user", "err", err) } ingesterQueryInterval, storeQueryInterval := q.buildQueryIntervals(params.Start, params.End) diff --git a/pkg/storage/stores/shipper/compactor/deletion/delete_requests_client.go b/pkg/storage/stores/shipper/compactor/deletion/delete_requests_client.go index bde272b520acc..f8626998f9ced 100644 --- a/pkg/storage/stores/shipper/compactor/deletion/delete_requests_client.go +++ b/pkg/storage/stores/shipper/compactor/deletion/delete_requests_client.go @@ -10,6 +10,7 @@ import ( "time" "github.com/go-kit/log/level" + "github.com/prometheus/client_golang/prometheus" "github.com/grafana/loki/pkg/util/log" ) @@ -32,6 +33,8 @@ type deleteRequestsClient struct { cache map[string][]DeleteRequest cacheDuration time.Duration + metrics *deleteRequestClientMetrics + stopChan chan struct{} } @@ -47,7 +50,7 @@ func WithRequestClientCacheDuration(d time.Duration) DeleteRequestsStoreOption { } } -func NewDeleteRequestsClient(addr string, c httpClient, opts ...DeleteRequestsStoreOption) (DeleteRequestsClient, error) { +func NewDeleteRequestsClient(addr string, c httpClient, r prometheus.Registerer, opts ...DeleteRequestsStoreOption) (DeleteRequestsClient, error) { u, err := url.Parse(addr) if err != nil { level.Error(log.Logger).Log("msg", "error parsing url", "err", err) @@ -60,6 +63,7 @@ func NewDeleteRequestsClient(addr string, c httpClient, opts ...DeleteRequestsSt httpClient: c, cacheDuration: 5 * time.Minute, cache: make(map[string][]DeleteRequest), + metrics: newDeleteRequestClientMetrics(r), stopChan: make(chan struct{}), } @@ -76,8 +80,10 @@ func (c *deleteRequestsClient) GetAllDeleteRequestsForUser(ctx context.Context, return cachedRequests, nil } + c.metrics.deleteRequestsLookupsTotal.Inc() requests, err := c.getRequestsFromServer(ctx, userID) if err != nil { + c.metrics.deleteRequestsLookupsFailedTotal.Inc() return nil, err } diff --git a/pkg/storage/stores/shipper/compactor/deletion/delete_requests_client_test.go b/pkg/storage/stores/shipper/compactor/deletion/delete_requests_client_test.go index f8f62ed964bea..5deed668570cc 100644 --- a/pkg/storage/stores/shipper/compactor/deletion/delete_requests_client_test.go +++ b/pkg/storage/stores/shipper/compactor/deletion/delete_requests_client_test.go @@ -14,7 +14,7 @@ import ( func TestGetCacheGenNumberForUser(t *testing.T) { t.Run("it requests results from the api", func(t *testing.T) { httpClient := &mockHTTPClient{ret: `[{"request_id":"test-request"}]`} - client, err := NewDeleteRequestsClient("http://test-server", httpClient) + client, err := NewDeleteRequestsClient("http://test-server", httpClient, nil) require.Nil(t, err) deleteRequests, err := client.GetAllDeleteRequestsForUser(context.Background(), "userID") @@ -30,7 +30,7 @@ func TestGetCacheGenNumberForUser(t *testing.T) { t.Run("it caches the results", func(t *testing.T) { httpClient := &mockHTTPClient{ret: `[{"request_id":"test-request"}]`} - client, err := NewDeleteRequestsClient("http://test-server", httpClient, WithRequestClientCacheDuration(100*time.Millisecond)) + client, err := NewDeleteRequestsClient("http://test-server", httpClient, nil, WithRequestClientCacheDuration(100*time.Millisecond)) require.Nil(t, err) deleteRequests, err := client.GetAllDeleteRequestsForUser(context.Background(), "userID") diff --git a/pkg/storage/stores/shipper/compactor/deletion/metrics.go b/pkg/storage/stores/shipper/compactor/deletion/metrics.go index 90b275e9bb84c..85a579d8c7145 100644 --- a/pkg/storage/stores/shipper/compactor/deletion/metrics.go +++ b/pkg/storage/stores/shipper/compactor/deletion/metrics.go @@ -5,6 +5,29 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) +type deleteRequestClientMetrics struct { + deleteRequestsLookupsTotal prometheus.Counter + deleteRequestsLookupsFailedTotal prometheus.Counter +} + +func newDeleteRequestClientMetrics(r prometheus.Registerer) *deleteRequestClientMetrics { + m := deleteRequestClientMetrics{} + + m.deleteRequestsLookupsTotal = promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: "loki", + Name: "delete_request_lookups_total", + Help: "Number times the client has looked up delete requests", + }) + + m.deleteRequestsLookupsFailedTotal = promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: "loki", + Name: "delete_request_lookups_failed_total", + Help: "Number times the client has failed to look up delete requests", + }) + + return &m +} + type deleteRequestHandlerMetrics struct { deleteRequestsReceivedTotal *prometheus.CounterVec }