Do not fail queries if compactor returns unexpected status code (#6552)…

… (#6683)

To avoid showing deleted data, the querier originally failed when
deletes were unavailable. If the compactor becomes unavailable this can
cause Loki to stop responding to queries altogether. This PR changes the
querier to log an error but continue servicing the request when it fails
to get deletes.

Extends PR #6368

Signed-off-by: Christian Haudum <christian.haudum@gmail.com>
(cherry picked from commit a1745c7)

Co-authored-by: Christian Haudum <christian.haudum@gmail.com>

pkg/querier/querier.go

@@ -123,7 +123,7 @@ func (q *SingleTenantQuerier) SelectLogs(ctx context.Context, params logql.Selec
 
 	params.QueryRequest.Deletes, err = q.deletesForUser(ctx, params.Start, params.End)
 	if err != nil {
-		return nil, err
+		level.Error(spanlogger.FromContext(ctx)).Log("msg", "failed loading deletes for user", "err", err)
 	}
 
 	ingesterQueryInterval, storeQueryInterval := q.buildQueryIntervals(params.Start, params.End)
@@ -411,7 +411,7 @@ func (q *SingleTenantQuerier) Tail(ctx context.Context, req *logproto.TailReques
 
 	deletes, err := q.deletesForUser(ctx, req.Start, time.Now())
 	if err != nil {
-		return nil, err
+		level.Error(spanlogger.FromContext(ctx)).Log("msg", "failed loading deletes for user", "err", err)
 	}
 
 	histReq := logql.SelectLogParams{