-
Notifications
You must be signed in to change notification settings - Fork 3.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
max line lengths (component + tenant overrides) (#1686)
* bytesize flag support * max line limits * configurable tenant override options for max line size * bytesize yaml support * validator methods * distributor validator type * human friendly line size output, distributor test * invalid labels distributor test * docs + vendor * removes dead limits code
- Loading branch information
Showing
17 changed files
with
640 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| package distributor | ||
|
|
||
| import "time" | ||
|
|
||
| // Limits is an interface for distributor limits/related configs | ||
| type Limits interface { | ||
| MaxLineSize(userID string) int | ||
| EnforceMetricName(userID string) bool | ||
| MaxLabelNamesPerSeries(userID string) int | ||
| MaxLabelNameLength(userID string) int | ||
| MaxLabelValueLength(userID string) int | ||
|
|
||
| CreationGracePeriod(userID string) time.Duration | ||
| RejectOldSamples(userID string) bool | ||
| RejectOldSamplesMaxAge(userID string) time.Duration | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| package distributor | ||
|
|
||
| import ( | ||
| "errors" | ||
| "net/http" | ||
| "time" | ||
|
|
||
| cortex_client "github.com/cortexproject/cortex/pkg/ingester/client" | ||
| cortex_validation "github.com/cortexproject/cortex/pkg/util/validation" | ||
| "github.com/grafana/loki/pkg/logproto" | ||
| "github.com/grafana/loki/pkg/util" | ||
| "github.com/grafana/loki/pkg/util/flagext" | ||
| "github.com/weaveworks/common/httpgrpc" | ||
| ) | ||
|
|
||
| type Validator struct { | ||
| Limits | ||
| } | ||
|
|
||
| func NewValidator(l Limits) (*Validator, error) { | ||
| if l == nil { | ||
| return nil, errors.New("nil Limits") | ||
| } | ||
| return &Validator{l}, nil | ||
| } | ||
|
|
||
| // ValidateEntry returns an error if the entry is invalid | ||
| func (v Validator) ValidateEntry(userID string, entry logproto.Entry) error { | ||
| if err := cortex_validation.ValidateSample(v, userID, metricName, cortex_client.Sample{ | ||
| TimestampMs: entry.Timestamp.UnixNano() / int64(time.Millisecond), | ||
| }); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if maxSize := v.MaxLineSize(userID); maxSize != 0 && len(entry.Line) > maxSize { | ||
| // I wish we didn't return httpgrpc errors here as it seems | ||
| // an orthogonal concept (we need not use ValidateLabels in this context) | ||
| // but the upstream cortex_validation pkg uses it, so we keep this | ||
| // for parity. | ||
| return httpgrpc.Errorf( | ||
| http.StatusBadRequest, | ||
| "max line size (%s) exceeded while adding (%s) size line", | ||
| flagext.ByteSize(uint64(maxSize)).String(), | ||
| flagext.ByteSize(uint64(len(entry.Line))).String(), | ||
| ) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // Validate labels returns an error if the labels are invalid | ||
| func (v Validator) ValidateLabels(userID string, labels string) error { | ||
| ls, err := util.ToClientLabels(labels) | ||
| if err != nil { | ||
| // I wish we didn't return httpgrpc errors here as it seems | ||
| // an orthogonal concept (we need not use ValidateLabels in this context) | ||
| // but the upstream cortex_validation pkg uses it, so we keep this | ||
| // for parity. | ||
| return httpgrpc.Errorf(http.StatusBadRequest, err.Error()) | ||
| } | ||
| return cortex_validation.ValidateLabels(v, userID, ls) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| package flagext | ||
|
|
||
| import ( | ||
| "strings" | ||
|
|
||
| "github.com/c2h5oh/datasize" | ||
| ) | ||
|
|
||
| // ByteSize is a flag parsing compatibility type for constructing human friendly sizes. | ||
| // It implements flag.Value & flag.Getter. | ||
| type ByteSize uint64 | ||
|
|
||
| func (bs ByteSize) String() string { | ||
| return datasize.ByteSize(bs).String() | ||
| } | ||
|
|
||
| func (bs *ByteSize) Set(s string) error { | ||
| var v datasize.ByteSize | ||
|
|
||
| // Bytesize currently doesn't handle things like Mb, but only handles MB. | ||
| // Therefore we capitalize just for convenience | ||
| if err := v.UnmarshalText([]byte(strings.ToUpper(s))); err != nil { | ||
| return err | ||
| } | ||
| *bs = ByteSize(v.Bytes()) | ||
| return nil | ||
| } | ||
|
|
||
| func (bs ByteSize) Get() interface{} { | ||
| return bs.Val() | ||
| } | ||
|
|
||
| func (bs ByteSize) Val() int { | ||
| return int(bs) | ||
| } | ||
|
|
||
| /// UnmarshalYAML the Unmarshaler interface of the yaml pkg. | ||
| func (bs *ByteSize) UnmarshalYAML(unmarshal func(interface{}) error) error { | ||
| var str string | ||
| err := unmarshal(&str) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return bs.Set(str) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| package flagext | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| "gopkg.in/yaml.v2" | ||
| ) | ||
|
|
||
| func Test_ByteSize(t *testing.T) { | ||
| for _, tc := range []struct { | ||
| in string | ||
| err bool | ||
| out int | ||
| }{ | ||
| { | ||
| in: "abc", | ||
| err: true, | ||
| }, | ||
| { | ||
| in: "", | ||
| err: false, | ||
| out: 0, | ||
| }, | ||
| { | ||
| in: "0", | ||
| err: false, | ||
| out: 0, | ||
| }, | ||
| { | ||
| in: "1b", | ||
| err: false, | ||
| out: 1, | ||
| }, | ||
| { | ||
| in: "100kb", | ||
| err: false, | ||
| out: 100 << 10, | ||
| }, | ||
| { | ||
| in: "100 KB", | ||
| err: false, | ||
| out: 100 << 10, | ||
| }, | ||
| { | ||
| // ensure lowercase works | ||
| in: "50mb", | ||
| err: false, | ||
| out: 50 << 20, | ||
| }, | ||
| { | ||
| // ensure mixed capitalization works | ||
| in: "50Mb", | ||
| err: false, | ||
| out: 50 << 20, | ||
| }, | ||
| { | ||
| in: "256GB", | ||
| err: false, | ||
| out: 256 << 30, | ||
| }, | ||
| } { | ||
| t.Run(tc.in, func(t *testing.T) { | ||
| var bs ByteSize | ||
|
|
||
| err := bs.Set(tc.in) | ||
| if tc.err { | ||
| require.NotNil(t, err) | ||
| } else { | ||
| require.Nil(t, err) | ||
| require.Equal(t, tc.out, bs.Get().(int)) | ||
| } | ||
|
|
||
| }) | ||
| } | ||
| } | ||
|
|
||
| func Test_ByteSizeYAML(t *testing.T) { | ||
| for _, tc := range []struct { | ||
| in string | ||
| err bool | ||
| out ByteSize | ||
| }{ | ||
| { | ||
| in: "256GB", | ||
| out: ByteSize(256 << 30), | ||
| }, | ||
| { | ||
| in: "abc", | ||
| err: true, | ||
| }, | ||
| } { | ||
| t.Run(tc.in, func(t *testing.T) { | ||
| var out ByteSize | ||
| err := yaml.Unmarshal([]byte(tc.in), &out) | ||
| if tc.err { | ||
| require.NotNil(t, err) | ||
| } else { | ||
| require.Nil(t, err) | ||
| require.Equal(t, tc.out, out) | ||
| } | ||
| }) | ||
| } | ||
| } |
Oops, something went wrong.