Revendor cortex to latest master

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
-	github.com/cortexproject/cortex v1.4.1-0.20201012150016-9e8beee8cacb
+	github.com/cortexproject/cortex v1.4.1-0.20201022071705-85942c5703cf
 	github.com/davecgh/go-spew v1.1.1
 	github.com/docker/docker v17.12.0-ce-rc1.0.20201009160326-9c15e82f19b0+incompatible
 	github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 // indirect
@@ -22,7 +22,7 @@ require (
 	github.com/go-logfmt/logfmt v0.5.0
 	github.com/gofrs/flock v0.7.1 // indirect
 	github.com/gogo/protobuf v1.3.1 // remember to update loki-build-image/Dockerfile too
-	github.com/golang/snappy v0.0.1
+	github.com/golang/snappy v0.0.2
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
@@ -32,7 +32,7 @@ require (
 	github.com/hpcloud/tail v1.0.0
 	github.com/imdario/mergo v0.3.9
 	github.com/influxdata/go-syslog/v3 v3.0.1-0.20200510134747-836dce2cf6da
-	github.com/jmespath/go-jmespath v0.3.0
+	github.com/jmespath/go-jmespath v0.4.0
 	github.com/joncrlsn/dque v2.2.1-0.20200515025108-956d14155fa2+incompatible
 	github.com/json-iterator/go v1.1.10
 	github.com/klauspost/compress v1.9.5
@@ -46,20 +46,20 @@ require (
 	github.com/prometheus/client_golang v1.7.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.14.0
-	github.com/prometheus/prometheus v1.8.2-0.20200923143134-7e2db3d092f3
+	github.com/prometheus/prometheus v1.8.2-0.20201014093524-73e2ce1bd643
 	github.com/segmentio/fasthash v1.0.2
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
-	github.com/shurcooL/vfsgen v0.0.0-20200627165143-92b8a710ab6c
+	github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546
 	github.com/stretchr/testify v1.6.1
 	github.com/tonistiigi/fifo v0.0.0-20190226154929-a9fb20d87448
 	github.com/uber/jaeger-client-go v2.25.0+incompatible
 	github.com/ugorji/go v1.1.7 // indirect
 	github.com/weaveworks/common v0.0.0-20200914083218-61ffdd448099
 	go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50
-	go.uber.org/atomic v1.6.0
-	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de
-	golang.org/x/net v0.0.0-20200822124328-c89045814202
-	google.golang.org/grpc v1.30.0
+	go.uber.org/atomic v1.7.0
+	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0
+	golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0
+	google.golang.org/grpc v1.32.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/fsnotify.v1 v1.4.7
 	gopkg.in/yaml.v2 v2.3.0
@@ -71,7 +71,8 @@ replace github.com/hpcloud/tail => github.com/grafana/tail v0.0.0-20201004203643
 
 replace github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v36.2.0+incompatible
 
-replace k8s.io/client-go => k8s.io/client-go v0.18.3
+// Keeping this same as Cortex to avoid dependency issues.
+replace k8s.io/client-go => k8s.io/client-go v0.19.2
 
 // >v1.2.0 has some conflict with prometheus/alertmanager. Hence prevent the upgrade till it's fixed.
 replace github.com/satori/go.uuid => github.com/satori/go.uuid v1.2.0

pkg/distributor/distributor_test.go

@@ -343,3 +343,7 @@ func (r mockRing) ShuffleShard(identifier string, size int) ring.ReadRing {
 	r.ingesters = r.ingesters[:size]
 	return r
 }
+
+func (r mockRing) ShuffleShardWithLookback(identifier string, size int, lookbackPeriod time.Duration, now time.Time) ring.ReadRing {
+	return r
+}

pkg/distributor/http.go

@@ -59,7 +59,7 @@ func ParseRequest(r *http.Request) (*logproto.PushRequest, error) {
 		}
 
 	default:
-		if _, err := util.ParseProtoReader(r.Context(), r.Body, int(r.ContentLength), math.MaxInt32, &req, util.RawSnappy); err != nil {
+		if err := util.ParseProtoReader(r.Context(), r.Body, int(r.ContentLength), math.MaxInt32, &req, util.RawSnappy); err != nil {
 			return nil, err
 		}
 	}

pkg/loki/loki.go

@@ -147,6 +147,7 @@ type Loki struct {
 	frontend        *frontend.Frontend
 	ruler           *cortex_ruler.Ruler
 	RulerStorage    rules.RuleStore
+	rulerAPI        *cortex_ruler.API
 	stopper         queryrange.Stopper
 	runtimeConfig   *runtimeconfig.Manager
 	memberlistKV    *memberlist.KVInitService

pkg/loki/modules.go

@@ -458,29 +458,31 @@ func (t *Loki) initRuler() (_ services.Service, err error) {
 		return
 	}
 
+	t.rulerAPI = cortex_ruler.NewAPI(t.ruler, t.RulerStorage)
+
 	// Expose HTTP endpoints.
 	if t.cfg.Ruler.EnableAPI {
 
 		t.server.HTTP.Handle("/ruler/ring", t.ruler)
 		cortex_ruler.RegisterRulerServer(t.server.GRPC, t.ruler)
 
 		// Prometheus Rule API Routes
-		t.server.HTTP.Path("/prometheus/api/v1/rules").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.ruler.PrometheusRules)))
-		t.server.HTTP.Path("/prometheus/api/v1/alerts").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.ruler.PrometheusAlerts)))
+		t.server.HTTP.Path("/prometheus/api/v1/rules").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.rulerAPI.PrometheusRules)))
+		t.server.HTTP.Path("/prometheus/api/v1/alerts").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.rulerAPI.PrometheusAlerts)))
 
 		// Ruler Legacy API Routes
-		t.server.HTTP.Path("/api/prom/rules").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.ruler.ListRules)))
-		t.server.HTTP.Path("/api/prom/rules/{namespace}").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.ruler.ListRules)))
-		t.server.HTTP.Path("/api/prom/rules/{namespace}/{groupName}").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.ruler.GetRuleGroup)))
-		t.server.HTTP.Path("/api/prom/rules/{namespace}").Methods("POST").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.ruler.CreateRuleGroup)))
-		t.server.HTTP.Path("/api/prom/rules/{namespace}/{groupName}").Methods("DELETE").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.ruler.DeleteRuleGroup)))
+		t.server.HTTP.Path("/api/prom/rules").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.rulerAPI.ListRules)))
+		t.server.HTTP.Path("/api/prom/rules/{namespace}").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.rulerAPI.ListRules)))
+		t.server.HTTP.Path("/api/prom/rules/{namespace}/{groupName}").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.rulerAPI.GetRuleGroup)))
+		t.server.HTTP.Path("/api/prom/rules/{namespace}").Methods("POST").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.rulerAPI.CreateRuleGroup)))
+		t.server.HTTP.Path("/api/prom/rules/{namespace}/{groupName}").Methods("DELETE").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.rulerAPI.DeleteRuleGroup)))
 
 		// Ruler API Routes
-		t.server.HTTP.Path("/loki/api/v1/rules").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.ruler.ListRules)))
-		t.server.HTTP.Path("/loki/api/v1/rules/{namespace}").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.ruler.ListRules)))
-		t.server.HTTP.Path("/loki/api/v1/rules/{namespace}/{groupName}").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.ruler.GetRuleGroup)))
-		t.server.HTTP.Path("/loki/api/v1/rules/{namespace}").Methods("POST").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.ruler.CreateRuleGroup)))
-		t.server.HTTP.Path("/loki/api/v1/rules/{namespace}/{groupName}").Methods("DELETE").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.ruler.DeleteRuleGroup)))
+		t.server.HTTP.Path("/loki/api/v1/rules").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.rulerAPI.ListRules)))
+		t.server.HTTP.Path("/loki/api/v1/rules/{namespace}").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.rulerAPI.ListRules)))
+		t.server.HTTP.Path("/loki/api/v1/rules/{namespace}/{groupName}").Methods("GET").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.rulerAPI.GetRuleGroup)))
+		t.server.HTTP.Path("/loki/api/v1/rules/{namespace}").Methods("POST").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.rulerAPI.CreateRuleGroup)))
+		t.server.HTTP.Path("/loki/api/v1/rules/{namespace}/{groupName}").Methods("DELETE").Handler(t.httpAuthMiddleware.Wrap(http.HandlerFunc(t.rulerAPI.DeleteRuleGroup)))
 	}
 
 	return t.ruler, nil

pkg/promtail/client/client_test.go

@@ -311,7 +311,7 @@ func createServerHandler(receivedReqsChan chan receivedReq, status int) http.Han
 	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 		// Parse the request
 		var pushReq logproto.PushRequest
-		if _, err := util.ParseProtoReader(req.Context(), req.Body, int(req.ContentLength), math.MaxInt32, &pushReq, util.RawSnappy); err != nil {
+		if err := util.ParseProtoReader(req.Context(), req.Body, int(req.ContentLength), math.MaxInt32, &pushReq, util.RawSnappy); err != nil {
 			rw.WriteHeader(500)
 			return
 		}

pkg/promtail/promtail_test.go

@@ -446,7 +446,7 @@ type testServerHandler struct {
 
 func (h *testServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	var req logproto.PushRequest
-	if _, err := util.ParseProtoReader(r.Context(), r.Body, int(r.ContentLength), math.MaxInt32, &req, util.RawSnappy); err != nil {
+	if err := util.ParseProtoReader(r.Context(), r.Body, int(r.ContentLength), math.MaxInt32, &req, util.RawSnappy); err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}

pkg/querier/querier_mock_test.go

@@ -344,6 +344,10 @@ func (r *readRingMock) HasInstance(instanceID string) bool {
 	return false
 }
 
+func (r *readRingMock) ShuffleShardWithLookback(identifier string, size int, lookbackPeriod time.Duration, now time.Time) ring.ReadRing {
+	return r
+}
+
 func mockReadRingWithOneActiveIngester() *readRingMock {
 	return newReadRingMock([]ring.IngesterDesc{
 		{Addr: "test", Timestamp: time.Now().UnixNano(), State: ring.ACTIVE, Tokens: []uint32{1, 2, 3}},

pkg/querier/queryrange/codec.go

@@ -306,6 +306,7 @@ func (codec) DecodeResponse(ctx context.Context, r *http.Response, req queryrang
 			Status:  resp.Status,
 			Version: uint32(loghttp.GetVersion(req.Path)),
 			Data:    data,
+			Headers: httpResponseHeadersToPromResponseHeaders(r.Header),
 		}, nil
 	case *LokiLabelNamesRequest:
 		var resp loghttp.LabelResponse
@@ -316,6 +317,7 @@ func (codec) DecodeResponse(ctx context.Context, r *http.Response, req queryrang
 			Status:  resp.Status,
 			Version: uint32(loghttp.GetVersion(req.Path)),
 			Data:    resp.Data,
+			Headers: httpResponseHeadersToPromResponseHeaders(r.Header),
 		}, nil
 	default:
 		var resp loghttp.QueryResponse
@@ -331,6 +333,7 @@ func (codec) DecodeResponse(ctx context.Context, r *http.Response, req queryrang
 						ResultType: loghttp.ResultTypeMatrix,
 						Result:     toProto(resp.Data.Result.(loghttp.Matrix)),
 					},
+					Headers: convertPrometheusResponseHeadersToPointers(httpResponseHeadersToPromResponseHeaders(r.Header)),
 				},
 				Statistics: resp.Data.Statistics,
 			}, nil
@@ -345,6 +348,7 @@ func (codec) DecodeResponse(ctx context.Context, r *http.Response, req queryrang
 					ResultType: loghttp.ResultTypeStream,
 					Result:     resp.Data.Result.(loghttp.Streams).ToProto(),
 				},
+				Headers: httpResponseHeadersToPromResponseHeaders(r.Header),
 			}, nil
 		default:
 			return nil, httpgrpc.Errorf(http.StatusBadRequest, "unsupported response type")
@@ -417,6 +421,8 @@ func (codec) EncodeResponse(ctx context.Context, res queryrange.Response) (*http
 	return &resp, nil
 }
 
+// NOTE: When we would start caching response from non-metric queries we would have to consider cache gen headers as well in
+// MergeResponse implementation for Loki codecs same as it is done in Cortex at https://github.com/cortexproject/cortex/blob/21bad57b346c730d684d6d0205efef133422ab28/pkg/querier/queryrange/query_range.go#L170
 func (codec) MergeResponse(responses ...queryrange.Response) (queryrange.Response, error) {
 	if len(responses) == 0 {
 		return nil, errors.New("merging responses requires at least one response")
@@ -668,3 +674,12 @@ func (p paramsWrapper) Limit() uint32 { return p.LokiRequest.Limit }
 func (p paramsWrapper) Shards() []string {
 	return p.LokiRequest.Shards
 }
+
+func httpResponseHeadersToPromResponseHeaders(httpHeaders http.Header) []queryrange.PrometheusResponseHeader {
+	var promHeaders []queryrange.PrometheusResponseHeader
+	for h, hv := range httpHeaders {
+		promHeaders = append(promHeaders, queryrange.PrometheusResponseHeader{Name: h, Values: hv})
+	}
+
+	return promHeaders
+}

pkg/querier/queryrange/codec_test.go

@@ -791,9 +791,10 @@ func Test_codec_MergeResponse(t *testing.T) {
 
 type badResponse struct{}
 
-func (badResponse) Reset()         {}
-func (badResponse) String() string { return "noop" }
-func (badResponse) ProtoMessage()  {}
+func (badResponse) Reset()                                             {}
+func (badResponse) String() string                                     { return "noop" }
+func (badResponse) ProtoMessage()                                      {}
+func (badResponse) GetHeaders() []*queryrange.PrometheusResponseHeader { return nil }
 
 type badReader struct{}