Skip to content

Commit

Permalink
[k159] Remove Prometheus dependency for push pkg (#9956)
Browse files Browse the repository at this point in the history
Backport 15af77b from #9937

---

**What this PR does / why we need it**:

In #9694, we modified the `push` pkg
to import Prometheus as a dependency to use the `labels.Labels` type for
the entries' non-indexed labels.

Having Prometheus as a dependency is problematic since any project
importing the `push` pkg will need to import Prometheus as a result. In
fact, this is one of the reasons why the `push` pkg was extracted from
Loki in the first place (#8259).

This PR removes the dependency of Prometheus from the `push` pkg by
copying some bits of the implementation for Prometheus' `labels.Labels`.
We copy:
- The Labels struct definition
- The JSON Marshaling and Unmarshaling methods for the labels. We need
this so labels are encoded as maps instead of an array of objects.

---

**Notes for reviewers:**

- To implement the JSON Marshaling and Unmarshaling methods the `push`
pkg now depends on `golang.org/x/exp`. I think it should be fine for
projects importing the `push` pkg to also depend on `golang.org/x/exp`,
right?

Co-authored-by: Salva Corts <salva.corts@grafana.com>
  • Loading branch information
grafanabot and salvacorts authored Jul 17, 2023
1 parent d8d5d87 commit 49b2b73
Show file tree
Hide file tree
Showing 10 changed files with 105 additions and 45 deletions.
5 changes: 2 additions & 3 deletions pkg/loghttp/query.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"github.com/buger/jsonparser"
json "github.com/json-iterator/go"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/model/labels"

"github.com/grafana/loki/pkg/logproto"
"github.com/grafana/loki/pkg/logqlmodel/stats"
Expand Down Expand Up @@ -157,12 +156,12 @@ func unmarshalHTTPToLogProtoEntry(data []byte) (logproto.Entry, error) {
}
e.Line = v
case 2: // nonIndexedLabels
var nonIndexedLabels labels.Labels
var nonIndexedLabels []logproto.LabelAdapter
err := jsonparser.ObjectEach(value, func(key, val []byte, dataType jsonparser.ValueType, _ int) error {
if dataType != jsonparser.String {
return jsonparser.MalformedStringError
}
nonIndexedLabels = append(nonIndexedLabels, labels.Label{
nonIndexedLabels = append(nonIndexedLabels, logproto.LabelAdapter{
Name: string(key),
Value: string(val),
})
Expand Down
4 changes: 2 additions & 2 deletions pkg/loghttp/query_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ func TestStreams_ToProto(t *testing.T) {
Labels: `{foo="bar"}`,
Entries: []logproto.Entry{
{Timestamp: time.Unix(0, 1), Line: "1"},
{Timestamp: time.Unix(0, 2), Line: "2", NonIndexedLabels: labels.Labels{
{Timestamp: time.Unix(0, 2), Line: "2", NonIndexedLabels: []logproto.LabelAdapter{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
}},
Expand All @@ -186,7 +186,7 @@ func TestStreams_ToProto(t *testing.T) {
Labels: `{foo="bar", lvl="error"}`,
Entries: []logproto.Entry{
{Timestamp: time.Unix(0, 3), Line: "3"},
{Timestamp: time.Unix(0, 4), Line: "4", NonIndexedLabels: labels.Labels{
{Timestamp: time.Unix(0, 4), Line: "4", NonIndexedLabels: []logproto.LabelAdapter{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
}},
Expand Down
8 changes: 2 additions & 6 deletions pkg/push/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,21 @@ go 1.19

require (
github.com/gogo/protobuf v1.3.2
github.com/prometheus/prometheus v0.43.1-0.20230419161410-69155c6ba1e9
github.com/stretchr/testify v1.8.2
golang.org/x/exp v0.0.0-20230321023759-10a507213a29
google.golang.org/grpc v1.53.0
)

require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect
github.com/kr/pretty v0.2.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
golang.org/x/exp v0.0.0-20230307190834-24139beb5833 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
google.golang.org/protobuf v1.29.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
14 changes: 3 additions & 11 deletions pkg/push/go.sum
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
Expand All @@ -11,8 +9,6 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww=
github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
Expand All @@ -23,10 +19,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
github.com/prometheus/prometheus v0.43.1-0.20230419161410-69155c6ba1e9 h1:GrpznPCSJgx8mGGj5qfKoHiou/dVx7uMce9/9rSdiuY=
github.com/prometheus/prometheus v0.43.1-0.20230419161410-69155c6ba1e9/go.mod h1:L8xLODXgpZM57D1MA7SPgsDecKj6ez4AF7mMczR1bis=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
Expand All @@ -39,8 +31,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20230307190834-24139beb5833 h1:SChBja7BCQewoTAU7IgvucQKMIXrEpFxNMs0spT3/5s=
golang.org/x/exp v0.0.0-20230307190834-24139beb5833/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw=
golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
Expand Down Expand Up @@ -79,7 +71,7 @@ google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62Uo
google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
52 changes: 45 additions & 7 deletions pkg/push/types.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
package push

import (
"encoding/json"
"fmt"
"io"
"strings"
"time"
"unsafe"

"github.com/prometheus/prometheus/model/labels"
"golang.org/x/exp/slices"
)

// Stream contains a unique labels set as a string and a set of entries for it.
Expand All @@ -23,10 +24,47 @@ type Stream struct {
type Entry struct {
Timestamp time.Time `protobuf:"bytes,1,opt,name=timestamp,proto3,stdtime" json:"ts"`
Line string `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
NonIndexedLabels labels.Labels `protobuf:"bytes,3,opt,name=nonIndexedLabels,proto3" json:"nonIndexedLabels,omitempty"`
NonIndexedLabels LabelsAdapter `protobuf:"bytes,3,opt,name=nonIndexedLabels,proto3" json:"nonIndexedLabels,omitempty"`
}

type LabelAdapter labels.Label
// LabelAdapter should be a copy of the Prometheus labels.Label type.
// We cannot import Prometheus in this package because it would create many dependencies
// in other projects importing this package. Instead, we copy the definition here, which should
// be kept in sync with the original so it can be casted to the prometheus type.
type LabelAdapter struct {
Name, Value string
}

// LabelsAdapter is equivalent to the Prometheus labels.Labels type.
type LabelsAdapter []LabelAdapter

// MarshalJSON implements json.Marshaler.
// Should be kept in sync with Prometheus's labels.Labels implementation.
func (ls LabelsAdapter) MarshalJSON() ([]byte, error) {
labelsMap := make(map[string]string, len(ls))
for _, l := range ls {
labelsMap[l.Name] = l.Value
}
return json.Marshal(labelsMap)
}

// UnmarshalJSON implements json.Unmarshaler.
// Should be kept in sync with Prometheus's labels.Labels implementation.
func (ls *LabelsAdapter) UnmarshalJSON(b []byte) error {
var m map[string]string

if err := json.Unmarshal(b, &m); err != nil {
return err
}

*ls = make([]LabelAdapter, 0, len(m))
for k, v := range m {
*ls = append(*ls, LabelAdapter{Name: k, Value: v})
}
slices.SortFunc(*ls, func(a, b LabelAdapter) bool { return a.Name < b.Name })

return nil
}

func (m *LabelAdapter) Marshal() (dAtA []byte, err error) {
size := m.Size()
Expand Down Expand Up @@ -428,8 +466,8 @@ func (m *Entry) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.NonIndexedLabels = append(m.NonIndexedLabels, labels.Label{})
if err := (*LabelAdapter)(&m.NonIndexedLabels[len(m.NonIndexedLabels)-1]).Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
m.NonIndexedLabels = append(m.NonIndexedLabels, LabelAdapter{})
if err := m.NonIndexedLabels[len(m.NonIndexedLabels)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
Expand Down Expand Up @@ -619,7 +657,7 @@ func (m *Entry) Size() (n int) {
}
if len(m.NonIndexedLabels) > 0 {
for _, e := range m.NonIndexedLabels {
l = (*LabelAdapter)(&e).Size()
l = e.Size()
n += 1 + l + sovPush(uint64(l))
}
}
Expand Down Expand Up @@ -702,7 +740,7 @@ func (m *Entry) Equal(that interface{}) bool {
return false
}
for i := range m.NonIndexedLabels {
if !(*LabelAdapter)(&m.NonIndexedLabels[i]).Equal(LabelAdapter(that1.NonIndexedLabels[i])) {
if !m.NonIndexedLabels[i].Equal(that1.NonIndexedLabels[i]) {
return false
}
}
Expand Down
5 changes: 2 additions & 3 deletions pkg/util/marshal/legacy/marshal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"time"

json "github.com/json-iterator/go"
"github.com/prometheus/prometheus/model/labels"
"github.com/stretchr/testify/require"

loghttp "github.com/grafana/loki/pkg/loghttp/legacy"
Expand All @@ -31,7 +30,7 @@ var queryTests = []struct {
{
Timestamp: mustParse(time.RFC3339Nano, "2019-09-13T18:32:23.380001319Z"),
Line: "super line with labels",
NonIndexedLabels: labels.Labels{
NonIndexedLabels: []logproto.LabelAdapter{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
},
Expand Down Expand Up @@ -184,7 +183,7 @@ var tailTests = []struct {
{
Timestamp: mustParse(time.RFC3339Nano, "2019-09-13T18:32:23.380001319Z"),
Line: "super line with labels",
NonIndexedLabels: labels.Labels{
NonIndexedLabels: []logproto.LabelAdapter{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
},
Expand Down
4 changes: 2 additions & 2 deletions pkg/util/marshal/marshal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ var queryTests = []struct {
{
Timestamp: time.Unix(0, 123456789012346),
Line: "super line with labels",
NonIndexedLabels: labels.Labels{
NonIndexedLabels: []logproto.LabelAdapter{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
},
Expand Down Expand Up @@ -500,7 +500,7 @@ var tailTests = []struct {
{
Timestamp: time.Unix(0, 123456789012346),
Line: "super line with labels",
NonIndexedLabels: labels.Labels{
NonIndexedLabels: []logproto.LabelAdapter{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
},
Expand Down
3 changes: 1 addition & 2 deletions pkg/util/unmarshal/legacy/unmarshal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"testing"
"time"

"github.com/prometheus/prometheus/model/labels"
"github.com/stretchr/testify/require"

"github.com/grafana/loki/pkg/logproto"
Expand All @@ -29,7 +28,7 @@ var pushTests = []struct {
{
Timestamp: mustParse(time.RFC3339Nano, "2019-09-13T18:32:23.380001319Z"),
Line: "super line with labels",
NonIndexedLabels: labels.Labels{
NonIndexedLabels: []logproto.LabelAdapter{
{Name: "a", Value: "1"},
{Name: "b", Value: "2"},
},
Expand Down
3 changes: 1 addition & 2 deletions pkg/util/unmarshal/unmarshal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"testing"
"time"

"github.com/prometheus/prometheus/model/labels"
"github.com/stretchr/testify/require"

"github.com/grafana/loki/pkg/loghttp"
Expand Down Expand Up @@ -53,7 +52,7 @@ var pushTests = []struct {
{
Timestamp: time.Unix(0, 123456789012345),
Line: "super line",
NonIndexedLabels: labels.Labels{
NonIndexedLabels: []logproto.LabelAdapter{
{Name: "a", Value: "1"},
{Name: "b", Value: "2"},
},
Expand Down
52 changes: 45 additions & 7 deletions vendor/github.com/grafana/loki/pkg/push/types.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 49b2b73

Please sign in to comment.