diff --git a/pkg/chunkenc/memchunk.go b/pkg/chunkenc/memchunk.go index 83a8df0f15cff..ae5b8f20c3d7d 100644 --- a/pkg/chunkenc/memchunk.go +++ b/pkg/chunkenc/memchunk.go @@ -1182,9 +1182,6 @@ func (si *bufferedIterator) Next() bool { si.Close() return false } - // we decode always the line length and ts as varint - si.stats.AddDecompressedBytes(int64(len(line)) + 2*binary.MaxVarintLen64) - si.stats.AddDecompressedLines(1) si.currTs = ts si.currLine = line @@ -1194,6 +1191,8 @@ func (si *bufferedIterator) Next() bool { // moveNext moves the buffer to the next entry func (si *bufferedIterator) moveNext() (int64, []byte, [][]byte, bool) { + var decompressedBytes int64 + var decompressedMetadataBytes int64 var ts int64 var tWidth, lWidth, lineSize, lastAttempt int for lWidth == 0 { // Read until both varints have enough bytes. @@ -1219,6 +1218,9 @@ func (si *bufferedIterator) moveNext() (int64, []byte, [][]byte, bool) { lastAttempt = si.readBufValid } + // TS and line length + decompressedBytes += 2 * binary.MaxVarintLen64 + if lineSize >= maxLineLength { si.err = fmt.Errorf("line too long %d, maximum %d", lineSize, maxLineLength) return 0, nil, nil, false @@ -1256,7 +1258,11 @@ func (si *bufferedIterator) moveNext() (int64, []byte, [][]byte, bool) { } } + decompressedBytes += int64(lineSize) + if si.format < chunkFormatV4 { + si.stats.AddDecompressedBytes(decompressedBytes) + si.stats.AddDecompressedLines(1) return ts, si.buf[:lineSize], nil, true } @@ -1286,6 +1292,9 @@ func (si *bufferedIterator) moveNext() (int64, []byte, [][]byte, bool) { lastAttempt = si.readBufValid } + // Number of labels + decompressedMetadataBytes += binary.MaxVarintLen64 + // Shift down what is still left in the fixed-size read buffer, if any. si.readBufValid = copy(si.readBuf[:], si.readBuf[labelsWidth:si.readBufValid]) @@ -1336,6 +1345,9 @@ func (si *bufferedIterator) moveNext() (int64, []byte, [][]byte, bool) { lastAttempt = si.readBufValid } + // Label size + decompressedMetadataBytes += binary.MaxVarintLen64 + // If the buffer is not yet initialize or too small, we get a new one. if si.metaLabelsBuf[i] == nil || labelSize > cap(si.metaLabelsBuf[i]) { // in case of a replacement we replace back the buffer in the pool @@ -1369,8 +1381,14 @@ func (si *bufferedIterator) moveNext() (int64, []byte, [][]byte, bool) { return 0, nil, nil, false } } + + decompressedMetadataBytes += int64(labelSize) } + si.stats.AddDecompressedLines(1) + si.stats.AddDecompressedNonIndexedLabelsBytes(decompressedMetadataBytes) + si.stats.AddDecompressedBytes(decompressedBytes + decompressedMetadataBytes) + return ts, si.buf[:lineSize], si.metaLabelsBuf[:metaLabelsBufLen], true } diff --git a/pkg/chunkenc/memchunk_test.go b/pkg/chunkenc/memchunk_test.go index dbde132512842..67bf4758a04f6 100644 --- a/pkg/chunkenc/memchunk_test.go +++ b/pkg/chunkenc/memchunk_test.go @@ -1574,10 +1574,26 @@ func TestMemChunk_IteratorWithNonIndexedLabels(t *testing.T) { labels.FromStrings("traceID", "123", "user", "d").String(), } + // The expected bytes is the sum of bytes decompressed and bytes read from the head chunk. + // First we add the bytes read from the store (aka decompressed). That's + // nonIndexedLabelsBytes = n. lines * (n. labels + (2 * n. labels) * (label length + label)) + // lineBytes = n. lines * (ts + line length + line) + expectedNonIndexedLabelsBytes := 2 * (binary.MaxVarintLen64 + (binary.MaxVarintLen64 + len("traceID") + binary.MaxVarintLen64 + len("123") + binary.MaxVarintLen64 + len("user") + binary.MaxVarintLen64 + len("a"))) + lineBytes := 2 * (2*binary.MaxVarintLen64 + len("lineA")) + // Now we add the bytes read from the head chunk. That's + // nonIndexedLabelsBytes = n. lines * (n. labels * (label name + label value)) + // lineBytes = n. lines * (line) + expectedNonIndexedLabelsBytes += 2 * (len("traceID") + len("789") + len("user") + len("c")) + lineBytes += 2 * (len("lineC")) + // Finally, the expected total bytes is the line bytes + non-indexed labels bytes + expectedBytes := lineBytes + expectedNonIndexedLabelsBytes + // We will run the test twice so the iterator will be created twice. // This is to ensure that the iterator is correctly closed. for i := 0; i < 2; i++ { - it, err := chk.Iterator(context.Background(), time.Unix(0, 0), time.Unix(0, math.MaxInt64), logproto.FORWARD, noopStreamPipeline) + sts, ctx := stats.NewContext(context.Background()) + + it, err := chk.Iterator(ctx, time.Unix(0, 0), time.Unix(0, math.MaxInt64), logproto.FORWARD, noopStreamPipeline) require.NoError(t, err) var lines []string @@ -1590,6 +1606,10 @@ func TestMemChunk_IteratorWithNonIndexedLabels(t *testing.T) { } assert.ElementsMatch(t, expectedLines, lines) assert.ElementsMatch(t, expectedStreams, streams) + + resultStats := sts.Result(0, 0, len(lines)) + require.Equal(t, int64(expectedBytes), resultStats.Summary.TotalBytesProcessed) + require.Equal(t, int64(expectedNonIndexedLabelsBytes), resultStats.Summary.TotalNonIndexedLabelsBytesProcessed) } }) } diff --git a/pkg/chunkenc/unordered.go b/pkg/chunkenc/unordered.go index 59dcc15174c23..4156f63ecc7a0 100644 --- a/pkg/chunkenc/unordered.go +++ b/pkg/chunkenc/unordered.go @@ -212,7 +212,14 @@ func (hb *unorderedHeadBlock) forEntries( for ; i < len(es.entries) && i >= 0; next() { line := es.entries[i].line metadataLabels := es.entries[i].metadataLabels - chunkStats.AddHeadChunkBytes(int64(len(line))) + + var nonIndexedLabelsBytes int64 + for _, label := range metadataLabels { + nonIndexedLabelsBytes += int64(len(label.Name) + len(label.Value)) + } + chunkStats.AddHeadChunkNonIndexedLabelsBytes(nonIndexedLabelsBytes) + chunkStats.AddHeadChunkBytes(int64(len(line)) + nonIndexedLabelsBytes) + err = entryFn(chunkStats, es.ts, line, metadataLabels) } diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index 5eb5e27c199a4..8caab7fe683d6 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -33,7 +33,12 @@ var ( bytesIngested = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "distributor_bytes_received_total", - Help: "The total number of uncompressed bytes received per tenant", + Help: "The total number of uncompressed bytes received per tenant. Includes non-indexed labels bytes.", + }, []string{"tenant", "retention_hours"}) + nonIndexedLabelsBytesIngested = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "loki", + Name: "distributor_non_indexed_labels_bytes_received_total", + Help: "The total number of uncompressed bytes received per tenant for entries metadata (non-indexed labels)", }, []string{"tenant", "retention_hours"}) linesIngested = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", @@ -41,8 +46,9 @@ var ( Help: "The total number of lines received per tenant", }, []string{"tenant"}) - bytesReceivedStats = analytics.NewCounter("distributor_bytes_received") - linesReceivedStats = analytics.NewCounter("distributor_lines_received") + bytesReceivedStats = analytics.NewCounter("distributor_bytes_received") + nonIndexedLabelsBytesReceivedStats = analytics.NewCounter("distributor_non_indexed_labels_bytes_received") + linesReceivedStats = analytics.NewCounter("distributor_lines_received") ) const applicationJSON = "application/json" @@ -141,7 +147,9 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete entriesSize += entrySize nonIndexedLabelsSize += entryLabelsSize bytesIngested.WithLabelValues(userID, retentionHours).Add(float64(entrySize)) + nonIndexedLabelsBytesIngested.WithLabelValues(userID, retentionHours).Add(float64(entryLabelsSize)) bytesReceivedStats.Inc(entrySize) + nonIndexedLabelsBytesReceivedStats.Inc(entryLabelsSize) if e.Timestamp.After(mostRecentEntry) { mostRecentEntry = e.Timestamp } diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index 8aa09e4a5212a..19e7124d66542 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -44,15 +44,16 @@ func deflateString(source string) string { } func TestParseRequest(t *testing.T) { - var previousBytesReceived, previousLinesReceived int + var previousBytesReceived, previousNonIndexedLabelsBytesReceived, previousLinesReceived int for index, test := range []struct { - path string - body string - contentType string - contentEncoding string - valid bool - expectedBytes int - expectedLines int + path string + body string + contentType string + contentEncoding string + valid bool + expectedNonIndexedLabelsBytes int + expectedBytes int + expectedLines int }{ { path: `/loki/api/v1/push`, @@ -176,16 +177,18 @@ func TestParseRequest(t *testing.T) { valid: false, }, { - path: `/loki/api/v1/push`, - body: deflateString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz", {"a": "a", "b": "b"} ] ] }]}`), - contentType: `application/json; charset=utf-8`, - contentEncoding: `deflate`, - valid: true, - expectedBytes: len("fizzbuzz") + 2*len("a") + 2*len("b"), - expectedLines: 1, + path: `/loki/api/v1/push`, + body: deflateString(`{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz", {"a": "a", "b": "b"} ] ] }]}`), + contentType: `application/json; charset=utf-8`, + contentEncoding: `deflate`, + valid: true, + expectedNonIndexedLabelsBytes: 2*len("a") + 2*len("b"), + expectedBytes: len("fizzbuzz") + 2*len("a") + 2*len("b"), + expectedLines: 1, }, } { t.Run(fmt.Sprintf("test %d", index), func(t *testing.T) { + nonIndexedLabelsBytesIngested.Reset() bytesIngested.Reset() linesIngested.Reset() @@ -199,6 +202,8 @@ func TestParseRequest(t *testing.T) { data, err := ParseRequest(util_log.Logger, "fake", request, nil) + nonIndexedLabelsBytesReceived := int(nonIndexedLabelsBytesReceivedStats.Value()["total"].(int64)) - previousNonIndexedLabelsBytesReceived + previousNonIndexedLabelsBytesReceived += nonIndexedLabelsBytesReceived bytesReceived := int(bytesReceivedStats.Value()["total"].(int64)) - previousBytesReceived previousBytesReceived += bytesReceived linesReceived := int(linesReceivedStats.Value()["total"].(int64)) - previousLinesReceived @@ -207,15 +212,19 @@ func TestParseRequest(t *testing.T) { if test.valid { assert.Nil(t, err, "Should not give error for %d", index) assert.NotNil(t, data, "Should give data for %d", index) + require.Equal(t, test.expectedNonIndexedLabelsBytes, nonIndexedLabelsBytesReceived) require.Equal(t, test.expectedBytes, bytesReceived) require.Equal(t, test.expectedLines, linesReceived) + require.Equal(t, float64(test.expectedNonIndexedLabelsBytes), testutil.ToFloat64(nonIndexedLabelsBytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedBytes), testutil.ToFloat64(bytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedLines), testutil.ToFloat64(linesIngested.WithLabelValues("fake"))) } else { assert.NotNil(t, err, "Should give error for %d", index) assert.Nil(t, data, "Should not give data for %d", index) + require.Equal(t, 0, nonIndexedLabelsBytesReceived) require.Equal(t, 0, bytesReceived) require.Equal(t, 0, linesReceived) + require.Equal(t, float64(0), testutil.ToFloat64(nonIndexedLabelsBytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(0), testutil.ToFloat64(bytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(0), testutil.ToFloat64(linesIngested.WithLabelValues("fake"))) } diff --git a/pkg/logqlmodel/stats/context.go b/pkg/logqlmodel/stats/context.go index 66814d3d457dd..824d2ef55a202 100644 --- a/pkg/logqlmodel/stats/context.go +++ b/pkg/logqlmodel/stats/context.go @@ -150,6 +150,8 @@ func JoinIngesters(ctx context.Context, inc Ingester) { func (r *Result) ComputeSummary(execTime time.Duration, queueTime time.Duration, totalEntriesReturned int) { r.Summary.TotalBytesProcessed = r.Querier.Store.Chunk.DecompressedBytes + r.Querier.Store.Chunk.HeadChunkBytes + r.Ingester.Store.Chunk.DecompressedBytes + r.Ingester.Store.Chunk.HeadChunkBytes + r.Summary.TotalNonIndexedLabelsBytesProcessed = r.Querier.Store.Chunk.DecompressedNonIndexedLabelsBytes + r.Querier.Store.Chunk.HeadChunkNonIndexedLabelsBytes + + r.Ingester.Store.Chunk.DecompressedNonIndexedLabelsBytes + r.Ingester.Store.Chunk.HeadChunkNonIndexedLabelsBytes r.Summary.TotalLinesProcessed = r.Querier.Store.Chunk.DecompressedLines + r.Querier.Store.Chunk.HeadChunkLines + r.Ingester.Store.Chunk.DecompressedLines + r.Ingester.Store.Chunk.HeadChunkLines r.Summary.TotalPostFilterLines = r.Querier.Store.Chunk.PostFilterLines + r.Ingester.Store.Chunk.PostFilterLines @@ -172,8 +174,10 @@ func (s *Store) Merge(m Store) { s.TotalChunksDownloaded += m.TotalChunksDownloaded s.ChunksDownloadTime += m.ChunksDownloadTime s.Chunk.HeadChunkBytes += m.Chunk.HeadChunkBytes + s.Chunk.HeadChunkNonIndexedLabelsBytes += m.Chunk.HeadChunkNonIndexedLabelsBytes s.Chunk.HeadChunkLines += m.Chunk.HeadChunkLines s.Chunk.DecompressedBytes += m.Chunk.DecompressedBytes + s.Chunk.DecompressedNonIndexedLabelsBytes += m.Chunk.DecompressedNonIndexedLabelsBytes s.Chunk.DecompressedLines += m.Chunk.DecompressedLines s.Chunk.CompressedBytes += m.Chunk.CompressedBytes s.Chunk.TotalDuplicates += m.Chunk.TotalDuplicates @@ -284,6 +288,10 @@ func (c *Context) AddHeadChunkBytes(i int64) { atomic.AddInt64(&c.store.Chunk.HeadChunkBytes, i) } +func (c *Context) AddHeadChunkNonIndexedLabelsBytes(i int64) { + atomic.AddInt64(&c.store.Chunk.HeadChunkNonIndexedLabelsBytes, i) +} + func (c *Context) AddCompressedBytes(i int64) { atomic.AddInt64(&c.store.Chunk.CompressedBytes, i) } @@ -292,6 +300,10 @@ func (c *Context) AddDecompressedBytes(i int64) { atomic.AddInt64(&c.store.Chunk.DecompressedBytes, i) } +func (c *Context) AddDecompressedNonIndexedLabelsBytes(i int64) { + atomic.AddInt64(&c.store.Chunk.DecompressedNonIndexedLabelsBytes, i) +} + func (c *Context) AddDecompressedLines(i int64) { atomic.AddInt64(&c.store.Chunk.DecompressedLines, i) } diff --git a/pkg/logqlmodel/stats/stats.pb.go b/pkg/logqlmodel/stats/stats.pb.go index 96822c643ecd3..8ebca564fe128 100644 --- a/pkg/logqlmodel/stats/stats.pb.go +++ b/pkg/logqlmodel/stats/stats.pb.go @@ -167,7 +167,7 @@ type Summary struct { BytesProcessedPerSecond int64 `protobuf:"varint,1,opt,name=bytesProcessedPerSecond,proto3" json:"bytesProcessedPerSecond"` // Total lines processed per second. LinesProcessedPerSecond int64 `protobuf:"varint,2,opt,name=linesProcessedPerSecond,proto3" json:"linesProcessedPerSecond"` - // Total bytes processed. + // Total bytes processed. Includes non-indexed labels bytes. TotalBytesProcessed int64 `protobuf:"varint,3,opt,name=totalBytesProcessed,proto3" json:"totalBytesProcessed"` // Total lines processed. TotalLinesProcessed int64 `protobuf:"varint,4,opt,name=totalLinesProcessed,proto3" json:"totalLinesProcessed"` @@ -190,6 +190,8 @@ type Summary struct { Shards int64 `protobuf:"varint,10,opt,name=shards,proto3" json:"shards"` // Total lines post query filtering TotalPostFilterLines int64 `protobuf:"varint,11,opt,name=totalPostFilterLines,proto3" json:"totalPostFilterLines"` + // Total bytes processed of metadata. + TotalNonIndexedLabelsBytesProcessed int64 `protobuf:"varint,12,opt,name=totalNonIndexedLabelsBytesProcessed,proto3" json:"totalNonIndexedLabelsBytesProcessed"` } func (m *Summary) Reset() { *m = Summary{} } @@ -301,6 +303,13 @@ func (m *Summary) GetTotalPostFilterLines() int64 { return 0 } +func (m *Summary) GetTotalNonIndexedLabelsBytesProcessed() int64 { + if m != nil { + return m.TotalNonIndexedLabelsBytesProcessed + } + return 0 +} + type Querier struct { Store Store `protobuf:"bytes,1,opt,name=store,proto3" json:"store"` } @@ -494,11 +503,11 @@ func (m *Store) GetChunk() Chunk { } type Chunk struct { - // Total bytes processed but was already in memory. (found in the headchunk) + // Total bytes processed but was already in memory (found in the headchunk). Includes non-indexed labels bytes. HeadChunkBytes int64 `protobuf:"varint,4,opt,name=headChunkBytes,proto3" json:"headChunkBytes"` // Total lines processed but was already in memory. (found in the headchunk) HeadChunkLines int64 `protobuf:"varint,5,opt,name=headChunkLines,proto3" json:"headChunkLines"` - // Total bytes decompressed and processed from chunks. + // Total bytes decompressed and processed from chunks. Includes non-indexed labels bytes. DecompressedBytes int64 `protobuf:"varint,6,opt,name=decompressedBytes,proto3" json:"decompressedBytes"` // Total lines decompressed and processed from chunks. DecompressedLines int64 `protobuf:"varint,7,opt,name=decompressedLines,proto3" json:"decompressedLines"` @@ -508,6 +517,10 @@ type Chunk struct { TotalDuplicates int64 `protobuf:"varint,9,opt,name=totalDuplicates,proto3" json:"totalDuplicates"` // Total lines post filtering PostFilterLines int64 `protobuf:"varint,10,opt,name=postFilterLines,proto3" json:"postFilterLines"` + // Total bytes processed for metadata but was already in memory. (found in the headchunk) + HeadChunkNonIndexedLabelsBytes int64 `protobuf:"varint,11,opt,name=headChunkNonIndexedLabelsBytes,proto3" json:"headChunkNonIndexedLabelsBytes"` + // Total bytes of entries metadata decompressed and processed from chunks. + DecompressedNonIndexedLabelsBytes int64 `protobuf:"varint,12,opt,name=decompressedNonIndexedLabelsBytes,proto3" json:"decompressedNonIndexedLabelsBytes"` } func (m *Chunk) Reset() { *m = Chunk{} } @@ -591,6 +604,20 @@ func (m *Chunk) GetPostFilterLines() int64 { return 0 } +func (m *Chunk) GetHeadChunkNonIndexedLabelsBytes() int64 { + if m != nil { + return m.HeadChunkNonIndexedLabelsBytes + } + return 0 +} + +func (m *Chunk) GetDecompressedNonIndexedLabelsBytes() int64 { + if m != nil { + return m.DecompressedNonIndexedLabelsBytes + } + return 0 +} + type Cache struct { EntriesFound int32 `protobuf:"varint,1,opt,name=entriesFound,proto3" json:"entriesFound"` EntriesRequested int32 `protobuf:"varint,2,opt,name=entriesRequested,proto3" json:"entriesRequested"` @@ -696,71 +723,75 @@ func init() { func init() { proto.RegisterFile("pkg/logqlmodel/stats/stats.proto", fileDescriptor_6cdfe5d2aea33ebb) } var fileDescriptor_6cdfe5d2aea33ebb = []byte{ - // 1011 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x56, 0xcb, 0x6f, 0xe3, 0x44, - 0x18, 0x8f, 0x13, 0x9c, 0xb4, 0xd3, 0xe7, 0x4e, 0xbb, 0xac, 0x01, 0xc9, 0xae, 0x72, 0xaa, 0x04, - 0x6a, 0xc4, 0x43, 0x42, 0x20, 0x56, 0x42, 0xee, 0x52, 0xa9, 0xd2, 0x22, 0xca, 0x57, 0xb8, 0x70, - 0x73, 0xec, 0xd9, 0xc4, 0xaa, 0x63, 0x27, 0x7e, 0xc0, 0xee, 0x8d, 0xff, 0x00, 0xfe, 0x0c, 0x2e, - 0xdc, 0xf9, 0x0b, 0xd0, 0x1e, 0x7b, 0xdc, 0x93, 0x45, 0xd3, 0x0b, 0xf8, 0xb4, 0x12, 0x77, 0x84, - 0xe6, 0x9b, 0x89, 0xed, 0x71, 0x1c, 0x69, 0x2f, 0xf1, 0x7c, 0xbf, 0xc7, 0xbc, 0xbf, 0x6f, 0x42, - 0x4e, 0xe6, 0x37, 0x93, 0x51, 0x10, 0x4d, 0x16, 0xc1, 0x2c, 0xf2, 0x58, 0x30, 0x4a, 0x52, 0x27, - 0x4d, 0xc4, 0xef, 0xd9, 0x3c, 0x8e, 0xd2, 0x88, 0xea, 0x18, 0xbc, 0x7b, 0x3c, 0x89, 0x26, 0x11, - 0x22, 0x23, 0xde, 0x12, 0xe4, 0xf0, 0x5f, 0x8d, 0xf4, 0x81, 0x25, 0x59, 0x90, 0xd2, 0xcf, 0xc8, - 0x20, 0xc9, 0x66, 0x33, 0x27, 0x7e, 0x61, 0x68, 0x27, 0xda, 0xe9, 0xce, 0x47, 0xfb, 0x67, 0xa2, - 0x9b, 0x6b, 0x81, 0xda, 0x07, 0x2f, 0x73, 0xab, 0x53, 0xe4, 0xd6, 0x4a, 0x06, 0xab, 0x06, 0xb7, - 0x2e, 0x32, 0x16, 0xfb, 0x2c, 0x36, 0xba, 0x8a, 0xf5, 0x5b, 0x81, 0x56, 0x56, 0x29, 0x83, 0x55, - 0x83, 0x3e, 0x26, 0x5b, 0x7e, 0x38, 0x61, 0x49, 0xca, 0x62, 0xa3, 0x87, 0xde, 0x03, 0xe9, 0xbd, - 0x94, 0xb0, 0x7d, 0x28, 0xcd, 0xa5, 0x10, 0xca, 0x16, 0xfd, 0x84, 0xf4, 0x5d, 0xc7, 0x9d, 0xb2, - 0xc4, 0x78, 0x0b, 0xcd, 0x7b, 0xd2, 0x7c, 0x8e, 0xa0, 0xbd, 0x27, 0xad, 0x3a, 0x8a, 0x40, 0x6a, - 0x87, 0xff, 0x68, 0xa4, 0x2f, 0x14, 0xf4, 0x43, 0xa2, 0xbb, 0xd3, 0x2c, 0xbc, 0x91, 0x6b, 0xde, - 0xad, 0xfb, 0x6b, 0x76, 0x2e, 0x01, 0xf1, 0xe1, 0x16, 0x3f, 0xf4, 0xd8, 0x73, 0xb9, 0xd6, 0x0d, - 0x16, 0x94, 0x80, 0xf8, 0xf0, 0x69, 0xc6, 0xb8, 0xcb, 0x72, 0x8d, 0xaa, 0x67, 0x5f, 0x7a, 0xa4, - 0x06, 0xe4, 0x97, 0x9e, 0x93, 0x1d, 0x94, 0x89, 0x03, 0x92, 0x2b, 0x54, 0xad, 0x47, 0xd2, 0x5a, - 0x17, 0x42, 0x3d, 0x18, 0xfe, 0xa1, 0x93, 0x81, 0x3c, 0x41, 0xfa, 0x3d, 0x79, 0x34, 0x7e, 0x91, - 0xb2, 0xe4, 0x2a, 0x8e, 0x5c, 0x96, 0x24, 0xcc, 0xbb, 0x62, 0xf1, 0x35, 0x73, 0xa3, 0xd0, 0xc3, - 0xe5, 0xf7, 0xec, 0xf7, 0x8a, 0xdc, 0xda, 0x24, 0x81, 0x4d, 0x04, 0xef, 0x36, 0xf0, 0xc3, 0xd6, - 0x6e, 0xbb, 0x55, 0xb7, 0x1b, 0x24, 0xb0, 0x89, 0xa0, 0x97, 0xe4, 0x28, 0x8d, 0x52, 0x27, 0xb0, - 0x95, 0x61, 0x71, 0x07, 0x7b, 0xf6, 0xa3, 0x22, 0xb7, 0xda, 0x68, 0x68, 0x03, 0xcb, 0xae, 0x9e, - 0x2a, 0x43, 0xe1, 0x8e, 0xd6, 0xbb, 0x52, 0x69, 0x68, 0x03, 0xe9, 0x29, 0xd9, 0x62, 0xcf, 0x99, - 0xfb, 0x9d, 0x3f, 0x63, 0x86, 0x7e, 0xa2, 0x9d, 0x6a, 0xf6, 0x2e, 0xbf, 0x9b, 0x2b, 0x0c, 0xca, - 0x16, 0x7d, 0x9f, 0x6c, 0x2f, 0x32, 0x96, 0x31, 0x94, 0xf6, 0x51, 0xba, 0x57, 0xe4, 0x56, 0x05, - 0x42, 0xd5, 0xa4, 0x67, 0x84, 0x24, 0xd9, 0x58, 0x64, 0x45, 0x62, 0x0c, 0x70, 0x62, 0xfb, 0x45, - 0x6e, 0xd5, 0x50, 0xa8, 0xb5, 0xe9, 0x53, 0x72, 0x8c, 0xb3, 0xfb, 0x2a, 0x4c, 0x91, 0x63, 0x69, - 0x16, 0x87, 0xcc, 0x33, 0xb6, 0xd0, 0x69, 0x14, 0xb9, 0xd5, 0xca, 0x43, 0x2b, 0x4a, 0x87, 0xa4, - 0x9f, 0xcc, 0x03, 0x3f, 0x4d, 0x8c, 0x6d, 0xf4, 0x13, 0x7e, 0x1b, 0x05, 0x02, 0xf2, 0x8b, 0x9a, - 0xa9, 0x13, 0x7b, 0x89, 0x41, 0x6a, 0x1a, 0x44, 0x40, 0x7e, 0xcb, 0x59, 0x5d, 0x45, 0x49, 0x7a, - 0xe1, 0x07, 0x29, 0x8b, 0x71, 0xf7, 0x8c, 0x9d, 0xc6, 0xac, 0x1a, 0x3c, 0xb4, 0xa2, 0xc3, 0x2f, - 0xc8, 0x40, 0x16, 0x10, 0x9e, 0x73, 0x49, 0x1a, 0xc5, 0xac, 0x91, 0xa6, 0xd7, 0x1c, 0xab, 0x72, - 0x0e, 0x25, 0x20, 0x3e, 0xc3, 0xdf, 0xbb, 0x64, 0xeb, 0xb2, 0xaa, 0x13, 0xbb, 0x38, 0x04, 0x30, - 0x9e, 0x36, 0xe2, 0xba, 0xeb, 0xf6, 0x61, 0x91, 0x5b, 0x0a, 0x0e, 0x4a, 0x44, 0x2f, 0x08, 0xc5, - 0xf8, 0x9c, 0xe7, 0x7d, 0xf2, 0xb5, 0x93, 0xa2, 0x57, 0xdc, 0xe9, 0xb7, 0x8b, 0xdc, 0x6a, 0x61, - 0xa1, 0x05, 0x2b, 0x47, 0xb7, 0x31, 0x4e, 0xe4, 0x15, 0xae, 0x46, 0x97, 0x38, 0x28, 0x11, 0xfd, - 0x9c, 0xec, 0x57, 0x17, 0xf0, 0x9a, 0x85, 0xa9, 0xbc, 0xaf, 0xb4, 0xc8, 0xad, 0x06, 0x03, 0x8d, - 0xb8, 0xda, 0x2f, 0xfd, 0x8d, 0xf7, 0xeb, 0x97, 0x2e, 0xd1, 0x91, 0x2f, 0x07, 0x16, 0x8b, 0x00, - 0xf6, 0x4c, 0x56, 0x87, 0x6a, 0xe0, 0x92, 0x81, 0x46, 0x4c, 0xbf, 0x21, 0x0f, 0x6b, 0xc8, 0x93, - 0xe8, 0xa7, 0x30, 0x88, 0x1c, 0xaf, 0xdc, 0xb5, 0x77, 0x8a, 0xdc, 0x6a, 0x17, 0x40, 0x3b, 0xcc, - 0xcf, 0xc0, 0x55, 0x30, 0x4c, 0xa7, 0x5e, 0x75, 0x06, 0xeb, 0x2c, 0xb4, 0x60, 0x55, 0xa1, 0x6f, - 0x94, 0x51, 0x8e, 0xb5, 0x17, 0xfa, 0xe1, 0x9f, 0x3d, 0xa2, 0x23, 0xcf, 0x77, 0x64, 0xca, 0x1c, - 0x4f, 0x88, 0x79, 0x69, 0xa9, 0x1f, 0x85, 0xca, 0x40, 0x23, 0x56, 0xbc, 0x22, 0x1b, 0xf4, 0x16, - 0xaf, 0xc8, 0x83, 0x46, 0x4c, 0xcf, 0xc9, 0x03, 0x8f, 0xb9, 0xd1, 0x6c, 0x1e, 0x63, 0xf1, 0x11, - 0x43, 0xf7, 0xd1, 0xfe, 0xb0, 0xc8, 0xad, 0x75, 0x12, 0xd6, 0xa1, 0x66, 0x27, 0x62, 0x0e, 0x83, - 0xf6, 0x4e, 0xc4, 0x34, 0xd6, 0x21, 0xfa, 0x98, 0x1c, 0x34, 0xe7, 0x21, 0x4a, 0xcd, 0x51, 0x91, - 0x5b, 0x4d, 0x0a, 0x9a, 0x00, 0xb7, 0xe3, 0xf1, 0x3e, 0xc9, 0xe6, 0x81, 0xef, 0x3a, 0xdc, 0xbe, - 0x5d, 0xd9, 0x1b, 0x14, 0x34, 0x01, 0x6e, 0x9f, 0x37, 0x4a, 0x0a, 0xa9, 0xec, 0x0d, 0x0a, 0x9a, - 0xc0, 0xf0, 0xbf, 0x2e, 0xd1, 0xf1, 0xbd, 0xe4, 0x99, 0xc8, 0x44, 0xed, 0xbb, 0x88, 0xb2, 0x50, - 0xa9, 0x03, 0x75, 0x1c, 0x94, 0x88, 0x7e, 0x49, 0x0e, 0xd9, 0xaa, 0x62, 0x2e, 0x32, 0x5e, 0x51, - 0xc4, 0x7d, 0xd6, 0xed, 0xe3, 0x22, 0xb7, 0xd6, 0x38, 0x58, 0x43, 0xe8, 0xa7, 0x64, 0x4f, 0x62, - 0x98, 0x62, 0xe2, 0x15, 0xd3, 0xed, 0x07, 0x45, 0x6e, 0xa9, 0x04, 0xa8, 0x21, 0x37, 0xe2, 0xb3, - 0x0b, 0xcc, 0x65, 0xfe, 0x8f, 0xe5, 0x9b, 0x85, 0x46, 0x85, 0x00, 0x35, 0xe4, 0xaf, 0x0f, 0x02, - 0x58, 0x38, 0xc4, 0x8d, 0xc3, 0xd7, 0xa7, 0x04, 0xa1, 0x6a, 0xf2, 0x47, 0x2d, 0x16, 0x73, 0x15, - 0xd7, 0x4b, 0x17, 0x8f, 0xda, 0x0a, 0x83, 0xb2, 0xc5, 0x37, 0xd0, 0xab, 0x27, 0xe2, 0xa0, 0x2a, - 0x65, 0x75, 0x1c, 0x94, 0xc8, 0x1e, 0xdf, 0xde, 0x99, 0x9d, 0x57, 0x77, 0x66, 0xe7, 0xf5, 0x9d, - 0xa9, 0xfd, 0xbc, 0x34, 0xb5, 0xdf, 0x96, 0xa6, 0xf6, 0x72, 0x69, 0x6a, 0xb7, 0x4b, 0x53, 0xfb, - 0x6b, 0x69, 0x6a, 0x7f, 0x2f, 0xcd, 0xce, 0xeb, 0xa5, 0xa9, 0xfd, 0x7a, 0x6f, 0x76, 0x6e, 0xef, - 0xcd, 0xce, 0xab, 0x7b, 0xb3, 0xf3, 0xc3, 0x07, 0x13, 0x3f, 0x9d, 0x66, 0xe3, 0x33, 0x37, 0x9a, - 0x8d, 0x26, 0xb1, 0xf3, 0xcc, 0x09, 0x9d, 0x51, 0x10, 0xdd, 0xf8, 0xa3, 0xb6, 0xff, 0xbc, 0xe3, - 0x3e, 0xfe, 0xa3, 0xfd, 0xf8, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0x48, 0x51, 0xf2, 0x09, 0x12, - 0x0b, 0x00, 0x00, + // 1083 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x57, 0xcd, 0x6f, 0xe3, 0x44, + 0x14, 0x8f, 0x5b, 0x9c, 0xb4, 0xd3, 0xcf, 0x9d, 0x76, 0x59, 0x03, 0x92, 0x5d, 0x82, 0x10, 0x95, + 0x40, 0x8d, 0xf8, 0x90, 0x10, 0x88, 0x95, 0x90, 0xbb, 0x54, 0xaa, 0x54, 0xa0, 0xbc, 0xc2, 0x85, + 0x9b, 0x63, 0xcf, 0xa6, 0xa6, 0x8e, 0x9d, 0x7a, 0x6c, 0xd8, 0xde, 0xf8, 0x0f, 0xe0, 0xbf, 0x80, + 0x0b, 0x27, 0xfe, 0x89, 0x3d, 0xf6, 0xb8, 0x27, 0x8b, 0xa6, 0x17, 0xf0, 0x69, 0x25, 0xee, 0x08, + 0xcd, 0x9b, 0x89, 0xbf, 0xe2, 0xa8, 0xbd, 0xc4, 0xf3, 0x7e, 0x1f, 0x33, 0xe3, 0x19, 0xbf, 0x37, + 0x13, 0xb2, 0x37, 0xb9, 0x18, 0x0d, 0x82, 0x68, 0x74, 0x19, 0x8c, 0x23, 0x8f, 0x05, 0x03, 0x9e, + 0x38, 0x09, 0x97, 0xbf, 0x07, 0x93, 0x38, 0x4a, 0x22, 0xaa, 0x63, 0xf0, 0xfa, 0xee, 0x28, 0x1a, + 0x45, 0x88, 0x0c, 0x44, 0x4b, 0x92, 0xfd, 0x7f, 0x35, 0xd2, 0x05, 0xc6, 0xd3, 0x20, 0xa1, 0x9f, + 0x90, 0x1e, 0x4f, 0xc7, 0x63, 0x27, 0xbe, 0x32, 0xb4, 0x3d, 0x6d, 0x7f, 0xed, 0x83, 0xcd, 0x03, + 0xd9, 0xcd, 0x99, 0x44, 0xed, 0xad, 0xe7, 0x99, 0xd5, 0xc9, 0x33, 0x6b, 0x26, 0x83, 0x59, 0x43, + 0x58, 0x2f, 0x53, 0x16, 0xfb, 0x2c, 0x36, 0x96, 0x6a, 0xd6, 0x6f, 0x24, 0x5a, 0x5a, 0x95, 0x0c, + 0x66, 0x0d, 0xfa, 0x98, 0xac, 0xf8, 0xe1, 0x88, 0xf1, 0x84, 0xc5, 0xc6, 0x32, 0x7a, 0xb7, 0x94, + 0xf7, 0x58, 0xc1, 0xf6, 0xb6, 0x32, 0x17, 0x42, 0x28, 0x5a, 0xf4, 0x23, 0xd2, 0x75, 0x1d, 0xf7, + 0x9c, 0x71, 0xe3, 0x15, 0x34, 0x6f, 0x28, 0xf3, 0x21, 0x82, 0xf6, 0x86, 0xb2, 0xea, 0x28, 0x02, + 0xa5, 0xed, 0xff, 0xa3, 0x91, 0xae, 0x54, 0xd0, 0xf7, 0x89, 0xee, 0x9e, 0xa7, 0xe1, 0x85, 0x7a, + 0xe7, 0xf5, 0xaa, 0xbf, 0x62, 0x17, 0x12, 0x90, 0x0f, 0x61, 0xf1, 0x43, 0x8f, 0x3d, 0x53, 0xef, + 0xba, 0xc0, 0x82, 0x12, 0x90, 0x0f, 0x31, 0xcd, 0x18, 0x57, 0x59, 0xbd, 0x63, 0xdd, 0xb3, 0xa9, + 0x3c, 0x4a, 0x03, 0xea, 0x49, 0x0f, 0xc9, 0x1a, 0xca, 0xe4, 0x06, 0xa9, 0x37, 0xac, 0x5b, 0x77, + 0x94, 0xb5, 0x2a, 0x84, 0x6a, 0xd0, 0xff, 0xb3, 0x4b, 0x7a, 0x6a, 0x07, 0xe9, 0x77, 0xe4, 0xd1, + 0xf0, 0x2a, 0x61, 0xfc, 0x34, 0x8e, 0x5c, 0xc6, 0x39, 0xf3, 0x4e, 0x59, 0x7c, 0xc6, 0xdc, 0x28, + 0xf4, 0xf0, 0xf5, 0x97, 0xed, 0x37, 0xf2, 0xcc, 0x5a, 0x24, 0x81, 0x45, 0x84, 0xe8, 0x36, 0xf0, + 0xc3, 0xd6, 0x6e, 0x97, 0xca, 0x6e, 0x17, 0x48, 0x60, 0x11, 0x41, 0x8f, 0xc9, 0x4e, 0x12, 0x25, + 0x4e, 0x60, 0xd7, 0x86, 0xc5, 0x15, 0x5c, 0xb6, 0x1f, 0xe5, 0x99, 0xd5, 0x46, 0x43, 0x1b, 0x58, + 0x74, 0x75, 0x52, 0x1b, 0x0a, 0x57, 0xb4, 0xda, 0x55, 0x9d, 0x86, 0x36, 0x90, 0xee, 0x93, 0x15, + 0xf6, 0x8c, 0xb9, 0xdf, 0xfa, 0x63, 0x66, 0xe8, 0x7b, 0xda, 0xbe, 0x66, 0xaf, 0x8b, 0x6f, 0x73, + 0x86, 0x41, 0xd1, 0xa2, 0xef, 0x92, 0xd5, 0xcb, 0x94, 0xa5, 0x0c, 0xa5, 0x5d, 0x94, 0x6e, 0xe4, + 0x99, 0x55, 0x82, 0x50, 0x36, 0xe9, 0x01, 0x21, 0x3c, 0x1d, 0xca, 0xac, 0xe0, 0x46, 0x0f, 0x27, + 0xb6, 0x99, 0x67, 0x56, 0x05, 0x85, 0x4a, 0x9b, 0x9e, 0x90, 0x5d, 0x9c, 0xdd, 0x17, 0x61, 0x82, + 0x1c, 0x4b, 0xd2, 0x38, 0x64, 0x9e, 0xb1, 0x82, 0x4e, 0x23, 0xcf, 0xac, 0x56, 0x1e, 0x5a, 0x51, + 0xda, 0x27, 0x5d, 0x3e, 0x09, 0xfc, 0x84, 0x1b, 0xab, 0xe8, 0x27, 0xe2, 0x6b, 0x94, 0x08, 0xa8, + 0x27, 0x6a, 0xce, 0x9d, 0xd8, 0xe3, 0x06, 0xa9, 0x68, 0x10, 0x01, 0xf5, 0x2c, 0x66, 0x75, 0x1a, + 0xf1, 0xe4, 0xc8, 0x0f, 0x12, 0x16, 0xe3, 0xea, 0x19, 0x6b, 0x8d, 0x59, 0x35, 0x78, 0x68, 0x45, + 0xe9, 0x15, 0x79, 0x0b, 0xf1, 0xaf, 0xa2, 0xf0, 0x58, 0xa4, 0x11, 0xf3, 0x4e, 0x9c, 0x21, 0x0b, + 0x78, 0xe3, 0x83, 0x58, 0xc7, 0xce, 0xdf, 0xc9, 0x33, 0xeb, 0x3e, 0x72, 0xb8, 0x8f, 0xa8, 0xff, + 0x19, 0xe9, 0xa9, 0xda, 0x25, 0xd2, 0x9d, 0x27, 0x51, 0xcc, 0x1a, 0x15, 0xe2, 0x4c, 0x60, 0x65, + 0xba, 0xa3, 0x04, 0xe4, 0xa3, 0xff, 0xc7, 0x12, 0x59, 0x39, 0x2e, 0x4b, 0xd4, 0x3a, 0x8e, 0x08, + 0x4c, 0x64, 0xac, 0xcc, 0x34, 0xdd, 0xde, 0xce, 0x33, 0xab, 0x86, 0x43, 0x2d, 0xa2, 0x47, 0x84, + 0x62, 0x7c, 0x28, 0x4a, 0x0e, 0xff, 0xd2, 0x49, 0xd0, 0x2b, 0xd3, 0xe9, 0xd5, 0x3c, 0xb3, 0x5a, + 0x58, 0x68, 0xc1, 0x8a, 0xd1, 0x6d, 0x8c, 0xb9, 0xca, 0x9e, 0x72, 0x74, 0x85, 0x43, 0x2d, 0xa2, + 0x9f, 0x92, 0xcd, 0xf2, 0xdb, 0x3f, 0x63, 0x61, 0xa2, 0x52, 0x85, 0xe6, 0x99, 0xd5, 0x60, 0xa0, + 0x11, 0x97, 0xeb, 0xa5, 0xdf, 0x7b, 0xbd, 0x7e, 0x59, 0x22, 0x3a, 0xf2, 0xc5, 0xc0, 0xf2, 0x25, + 0x80, 0x3d, 0x55, 0x85, 0xa9, 0x1c, 0xb8, 0x60, 0xa0, 0x11, 0xd3, 0xaf, 0xc9, 0xc3, 0x0a, 0xf2, + 0x24, 0xfa, 0x29, 0x0c, 0x22, 0xc7, 0x2b, 0x56, 0xed, 0xb5, 0x3c, 0xb3, 0xda, 0x05, 0xd0, 0x0e, + 0x8b, 0x3d, 0x70, 0x6b, 0x18, 0x66, 0xf2, 0x72, 0xb9, 0x07, 0xf3, 0x2c, 0xb4, 0x60, 0xe5, 0x19, + 0xd3, 0xa8, 0xe0, 0x02, 0x6b, 0x3f, 0x63, 0xfa, 0xbf, 0xe9, 0x44, 0x47, 0x5e, 0xac, 0xc8, 0x39, + 0x73, 0x3c, 0x29, 0x16, 0x1f, 0x69, 0x75, 0x2b, 0xea, 0x0c, 0x34, 0xe2, 0x9a, 0x57, 0x26, 0xa2, + 0xde, 0xe2, 0x95, 0x29, 0xd8, 0x88, 0xe9, 0x21, 0x79, 0xe0, 0x31, 0x37, 0x1a, 0x4f, 0x62, 0xcc, + 0x08, 0x39, 0x74, 0x17, 0xed, 0x0f, 0xf3, 0xcc, 0x9a, 0x27, 0x61, 0x1e, 0x6a, 0x76, 0x22, 0xe7, + 0xd0, 0x6b, 0xef, 0x44, 0x4e, 0x63, 0x1e, 0xa2, 0x8f, 0xc9, 0x56, 0x73, 0x1e, 0xb2, 0xca, 0xed, + 0xe4, 0x99, 0xd5, 0xa4, 0xa0, 0x09, 0x08, 0x3b, 0x6e, 0xef, 0x93, 0x74, 0x12, 0xf8, 0xae, 0x23, + 0xec, 0xab, 0xa5, 0xbd, 0x41, 0x41, 0x13, 0x10, 0xf6, 0x49, 0xa3, 0x9a, 0x91, 0xd2, 0xde, 0xa0, + 0xa0, 0x09, 0xd0, 0x1f, 0x88, 0x59, 0x2c, 0x6c, 0x6b, 0xcd, 0x51, 0xb5, 0xb1, 0x9f, 0x67, 0xd6, + 0x1d, 0x4a, 0xb8, 0x83, 0xa7, 0x9c, 0xbc, 0x59, 0x5d, 0xbd, 0xf6, 0xe1, 0x64, 0xb5, 0x7c, 0x3b, + 0xcf, 0xac, 0xbb, 0xc5, 0x70, 0xb7, 0xa4, 0xff, 0xdf, 0x12, 0xd1, 0xf1, 0x2e, 0x22, 0x4a, 0x0d, + 0x93, 0xe7, 0xca, 0x51, 0x94, 0x86, 0xb5, 0x42, 0x57, 0xc5, 0xa1, 0x16, 0xd1, 0xcf, 0xc9, 0x36, + 0x9b, 0x9d, 0x46, 0x97, 0xa9, 0x28, 0x99, 0x32, 0x61, 0x75, 0x7b, 0x37, 0xcf, 0xac, 0x39, 0x0e, + 0xe6, 0x10, 0xfa, 0x31, 0xd9, 0x50, 0x18, 0xd6, 0x10, 0x79, 0x43, 0xd0, 0xed, 0x07, 0x79, 0x66, + 0xd5, 0x09, 0xa8, 0x87, 0xc2, 0x88, 0x57, 0x1a, 0x60, 0x2e, 0xf3, 0x7f, 0x2c, 0xee, 0x03, 0x68, + 0xac, 0x11, 0x50, 0x0f, 0xc5, 0xc9, 0x8e, 0x00, 0x56, 0x46, 0x99, 0x52, 0x78, 0xb2, 0x17, 0x20, + 0x94, 0x4d, 0x71, 0x61, 0x88, 0xe5, 0x5c, 0x65, 0xfe, 0xe8, 0xf2, 0xc2, 0x30, 0xc3, 0xa0, 0x68, + 0x89, 0x05, 0xf4, 0xaa, 0x95, 0xa6, 0x57, 0xd6, 0xea, 0x2a, 0x0e, 0xb5, 0xc8, 0x1e, 0x5e, 0xdf, + 0x98, 0x9d, 0x17, 0x37, 0x66, 0xe7, 0xe5, 0x8d, 0xa9, 0xfd, 0x3c, 0x35, 0xb5, 0xdf, 0xa7, 0xa6, + 0xf6, 0x7c, 0x6a, 0x6a, 0xd7, 0x53, 0x53, 0xfb, 0x6b, 0x6a, 0x6a, 0x7f, 0x4f, 0xcd, 0xce, 0xcb, + 0xa9, 0xa9, 0xfd, 0x7a, 0x6b, 0x76, 0xae, 0x6f, 0xcd, 0xce, 0x8b, 0x5b, 0xb3, 0xf3, 0xfd, 0x7b, + 0x23, 0x3f, 0x39, 0x4f, 0x87, 0x07, 0x6e, 0x34, 0x1e, 0x8c, 0x62, 0xe7, 0xa9, 0x13, 0x3a, 0x83, + 0x20, 0xba, 0xf0, 0x07, 0x6d, 0xff, 0x27, 0x86, 0x5d, 0xfc, 0xb7, 0xf0, 0xe1, 0xff, 0x01, 0x00, + 0x00, 0xff, 0xff, 0xe6, 0xbb, 0xa2, 0x88, 0x6e, 0x0c, 0x00, 0x00, } func (this *Result) Equal(that interface{}) bool { @@ -881,6 +912,9 @@ func (this *Summary) Equal(that interface{}) bool { if this.TotalPostFilterLines != that1.TotalPostFilterLines { return false } + if this.TotalNonIndexedLabelsBytesProcessed != that1.TotalNonIndexedLabelsBytesProcessed { + return false + } return true } func (this *Querier) Equal(that interface{}) bool { @@ -1016,6 +1050,12 @@ func (this *Chunk) Equal(that interface{}) bool { if this.PostFilterLines != that1.PostFilterLines { return false } + if this.HeadChunkNonIndexedLabelsBytes != that1.HeadChunkNonIndexedLabelsBytes { + return false + } + if this.DecompressedNonIndexedLabelsBytes != that1.DecompressedNonIndexedLabelsBytes { + return false + } return true } func (this *Cache) Equal(that interface{}) bool { @@ -1090,7 +1130,7 @@ func (this *Summary) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 15) + s := make([]string, 0, 16) s = append(s, "&stats.Summary{") s = append(s, "BytesProcessedPerSecond: "+fmt.Sprintf("%#v", this.BytesProcessedPerSecond)+",\n") s = append(s, "LinesProcessedPerSecond: "+fmt.Sprintf("%#v", this.LinesProcessedPerSecond)+",\n") @@ -1103,6 +1143,7 @@ func (this *Summary) GoString() string { s = append(s, "Splits: "+fmt.Sprintf("%#v", this.Splits)+",\n") s = append(s, "Shards: "+fmt.Sprintf("%#v", this.Shards)+",\n") s = append(s, "TotalPostFilterLines: "+fmt.Sprintf("%#v", this.TotalPostFilterLines)+",\n") + s = append(s, "TotalNonIndexedLabelsBytesProcessed: "+fmt.Sprintf("%#v", this.TotalNonIndexedLabelsBytesProcessed)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -1147,7 +1188,7 @@ func (this *Chunk) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 11) + s := make([]string, 0, 13) s = append(s, "&stats.Chunk{") s = append(s, "HeadChunkBytes: "+fmt.Sprintf("%#v", this.HeadChunkBytes)+",\n") s = append(s, "HeadChunkLines: "+fmt.Sprintf("%#v", this.HeadChunkLines)+",\n") @@ -1156,6 +1197,8 @@ func (this *Chunk) GoString() string { s = append(s, "CompressedBytes: "+fmt.Sprintf("%#v", this.CompressedBytes)+",\n") s = append(s, "TotalDuplicates: "+fmt.Sprintf("%#v", this.TotalDuplicates)+",\n") s = append(s, "PostFilterLines: "+fmt.Sprintf("%#v", this.PostFilterLines)+",\n") + s = append(s, "HeadChunkNonIndexedLabelsBytes: "+fmt.Sprintf("%#v", this.HeadChunkNonIndexedLabelsBytes)+",\n") + s = append(s, "DecompressedNonIndexedLabelsBytes: "+fmt.Sprintf("%#v", this.DecompressedNonIndexedLabelsBytes)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -1329,6 +1372,11 @@ func (m *Summary) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.TotalNonIndexedLabelsBytesProcessed != 0 { + i = encodeVarintStats(dAtA, i, uint64(m.TotalNonIndexedLabelsBytesProcessed)) + i-- + dAtA[i] = 0x60 + } if m.TotalPostFilterLines != 0 { i = encodeVarintStats(dAtA, i, uint64(m.TotalPostFilterLines)) i-- @@ -1543,6 +1591,16 @@ func (m *Chunk) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.DecompressedNonIndexedLabelsBytes != 0 { + i = encodeVarintStats(dAtA, i, uint64(m.DecompressedNonIndexedLabelsBytes)) + i-- + dAtA[i] = 0x60 + } + if m.HeadChunkNonIndexedLabelsBytes != 0 { + i = encodeVarintStats(dAtA, i, uint64(m.HeadChunkNonIndexedLabelsBytes)) + i-- + dAtA[i] = 0x58 + } if m.PostFilterLines != 0 { i = encodeVarintStats(dAtA, i, uint64(m.PostFilterLines)) i-- @@ -1723,6 +1781,9 @@ func (m *Summary) Size() (n int) { if m.TotalPostFilterLines != 0 { n += 1 + sovStats(uint64(m.TotalPostFilterLines)) } + if m.TotalNonIndexedLabelsBytesProcessed != 0 { + n += 1 + sovStats(uint64(m.TotalNonIndexedLabelsBytesProcessed)) + } return n } @@ -1807,6 +1868,12 @@ func (m *Chunk) Size() (n int) { if m.PostFilterLines != 0 { n += 1 + sovStats(uint64(m.PostFilterLines)) } + if m.HeadChunkNonIndexedLabelsBytes != 0 { + n += 1 + sovStats(uint64(m.HeadChunkNonIndexedLabelsBytes)) + } + if m.DecompressedNonIndexedLabelsBytes != 0 { + n += 1 + sovStats(uint64(m.DecompressedNonIndexedLabelsBytes)) + } return n } @@ -1888,6 +1955,7 @@ func (this *Summary) String() string { `Splits:` + fmt.Sprintf("%v", this.Splits) + `,`, `Shards:` + fmt.Sprintf("%v", this.Shards) + `,`, `TotalPostFilterLines:` + fmt.Sprintf("%v", this.TotalPostFilterLines) + `,`, + `TotalNonIndexedLabelsBytesProcessed:` + fmt.Sprintf("%v", this.TotalNonIndexedLabelsBytesProcessed) + `,`, `}`, }, "") return s @@ -1941,6 +2009,8 @@ func (this *Chunk) String() string { `CompressedBytes:` + fmt.Sprintf("%v", this.CompressedBytes) + `,`, `TotalDuplicates:` + fmt.Sprintf("%v", this.TotalDuplicates) + `,`, `PostFilterLines:` + fmt.Sprintf("%v", this.PostFilterLines) + `,`, + `HeadChunkNonIndexedLabelsBytes:` + fmt.Sprintf("%v", this.HeadChunkNonIndexedLabelsBytes) + `,`, + `DecompressedNonIndexedLabelsBytes:` + fmt.Sprintf("%v", this.DecompressedNonIndexedLabelsBytes) + `,`, `}`, }, "") return s @@ -2561,6 +2631,25 @@ func (m *Summary) Unmarshal(dAtA []byte) error { break } } + case 12: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TotalNonIndexedLabelsBytesProcessed", wireType) + } + m.TotalNonIndexedLabelsBytesProcessed = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStats + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TotalNonIndexedLabelsBytesProcessed |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipStats(dAtA[iNdEx:]) @@ -3138,6 +3227,44 @@ func (m *Chunk) Unmarshal(dAtA []byte) error { break } } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field HeadChunkNonIndexedLabelsBytes", wireType) + } + m.HeadChunkNonIndexedLabelsBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStats + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.HeadChunkNonIndexedLabelsBytes |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 12: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DecompressedNonIndexedLabelsBytes", wireType) + } + m.DecompressedNonIndexedLabelsBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStats + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.DecompressedNonIndexedLabelsBytes |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipStats(dAtA[iNdEx:]) diff --git a/pkg/logqlmodel/stats/stats.proto b/pkg/logqlmodel/stats/stats.proto index 2913620db3f4e..019bc396519de 100644 --- a/pkg/logqlmodel/stats/stats.proto +++ b/pkg/logqlmodel/stats/stats.proto @@ -53,7 +53,7 @@ message Summary { int64 bytesProcessedPerSecond = 1 [(gogoproto.jsontag) = "bytesProcessedPerSecond"]; // Total lines processed per second. int64 linesProcessedPerSecond = 2 [(gogoproto.jsontag) = "linesProcessedPerSecond"]; - // Total bytes processed. + // Total bytes processed. Includes non-indexed labels bytes. int64 totalBytesProcessed = 3 [(gogoproto.jsontag) = "totalBytesProcessed"]; // Total lines processed. int64 totalLinesProcessed = 4 [(gogoproto.jsontag) = "totalLinesProcessed"]; @@ -76,6 +76,8 @@ message Summary { int64 shards = 10 [(gogoproto.jsontag) = "shards"]; // Total lines post query filtering int64 totalPostFilterLines = 11 [(gogoproto.jsontag) = "totalPostFilterLines"]; + // Total bytes processed of metadata. + int64 totalNonIndexedLabelsBytesProcessed = 12 [(gogoproto.jsontag) = "totalNonIndexedLabelsBytesProcessed"]; } message Querier { @@ -116,11 +118,11 @@ message Store { } message Chunk { - // Total bytes processed but was already in memory. (found in the headchunk) + // Total bytes processed but was already in memory (found in the headchunk). Includes non-indexed labels bytes. int64 headChunkBytes = 4 [(gogoproto.jsontag) = "headChunkBytes"]; // Total lines processed but was already in memory. (found in the headchunk) int64 headChunkLines = 5 [(gogoproto.jsontag) = "headChunkLines"]; - // Total bytes decompressed and processed from chunks. + // Total bytes decompressed and processed from chunks. Includes non-indexed labels bytes. int64 decompressedBytes = 6 [(gogoproto.jsontag) = "decompressedBytes"]; // Total lines decompressed and processed from chunks. int64 decompressedLines = 7 [(gogoproto.jsontag) = "decompressedLines"]; @@ -130,6 +132,10 @@ message Chunk { int64 totalDuplicates = 9 [(gogoproto.jsontag) = "totalDuplicates"]; // Total lines post filtering int64 postFilterLines = 10 [(gogoproto.jsontag) = "postFilterLines"]; + // Total bytes processed for metadata but was already in memory. (found in the headchunk) + int64 headChunkNonIndexedLabelsBytes = 11 [(gogoproto.jsontag) = "headChunkNonIndexedLabelsBytes"]; + // Total bytes of entries metadata decompressed and processed from chunks. + int64 decompressedNonIndexedLabelsBytes = 12 [(gogoproto.jsontag) = "decompressedNonIndexedLabelsBytes"]; } message Cache { diff --git a/pkg/querier/http_test.go b/pkg/querier/http_test.go index 5a206b358da72..41a51a531ab95 100644 --- a/pkg/querier/http_test.go +++ b/pkg/querier/http_test.go @@ -32,8 +32,10 @@ var ( "compressedBytes": 1, "decompressedBytes": 2, "decompressedLines": 3, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 4, "headChunkLines": 5, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 8 }, @@ -52,8 +54,10 @@ var ( "compressedBytes": 11, "decompressedBytes": 12, "decompressedLines": 13, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 14, "headChunkLines": 15, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 19 }, @@ -111,6 +115,7 @@ var ( "totalBytesProcessed": 24, "totalEntriesReturned": 10, "totalLinesProcessed": 25, + "totalNonIndexedLabelsBytesProcessed": 0, "totalPostFilterLines": 0 } }` diff --git a/pkg/querier/queryrange/codec_test.go b/pkg/querier/queryrange/codec_test.go index 6a62826f56f8d..456a3b9fa9188 100644 --- a/pkg/querier/queryrange/codec_test.go +++ b/pkg/querier/queryrange/codec_test.go @@ -1325,8 +1325,10 @@ var ( "compressedBytes": 1, "decompressedBytes": 2, "decompressedLines": 3, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 4, "headChunkLines": 5, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 8 }, @@ -1345,8 +1347,10 @@ var ( "compressedBytes": 11, "decompressedBytes": 12, "decompressedLines": 13, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 14, "headChunkLines": 15, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 19 }, @@ -1404,6 +1408,7 @@ var ( "totalBytesProcessed": 24, "totalEntriesReturned": 10, "totalLinesProcessed": 25, + "totalNonIndexedLabelsBytesProcessed": 0, "totalPostFilterLines": 0 } },` diff --git a/pkg/querier/queryrange/prometheus_test.go b/pkg/querier/queryrange/prometheus_test.go index 6b74746cc085d..ff194d94611a8 100644 --- a/pkg/querier/queryrange/prometheus_test.go +++ b/pkg/querier/queryrange/prometheus_test.go @@ -22,8 +22,10 @@ var emptyStats = `"stats": { "compressedBytes": 0, "decompressedBytes": 0, "decompressedLines": 0, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 0, "headChunkLines": 0, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 0 } @@ -42,8 +44,10 @@ var emptyStats = `"stats": { "compressedBytes": 0, "decompressedBytes": 0, "decompressedLines": 0, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 0, "headChunkLines": 0, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 0 } @@ -98,6 +102,7 @@ var emptyStats = `"stats": { "totalBytesProcessed":0, "totalEntriesReturned":0, "totalLinesProcessed":0, + "totalNonIndexedLabelsBytesProcessed": 0, "totalPostFilterLines": 0 } }` diff --git a/pkg/util/marshal/legacy/marshal_test.go b/pkg/util/marshal/legacy/marshal_test.go index d78b0b3adf93b..0e0bba91214d8 100644 --- a/pkg/util/marshal/legacy/marshal_test.go +++ b/pkg/util/marshal/legacy/marshal_test.go @@ -69,8 +69,10 @@ var queryTests = []struct { "compressedBytes": 0, "decompressedBytes": 0, "decompressedLines": 0, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 0, "headChunkLines": 0, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 0 } @@ -89,8 +91,10 @@ var queryTests = []struct { "compressedBytes": 0, "decompressedBytes": 0, "decompressedLines": 0, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 0, "headChunkLines": 0, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 0 } @@ -145,6 +149,7 @@ var queryTests = []struct { "totalBytesProcessed": 0, "totalEntriesReturned": 0, "totalLinesProcessed": 0, + "totalNonIndexedLabelsBytesProcessed": 0, "totalPostFilterLines": 0 } } diff --git a/pkg/util/marshal/marshal_test.go b/pkg/util/marshal/marshal_test.go index b1b9f66368ccd..2ebc27983ba7f 100644 --- a/pkg/util/marshal/marshal_test.go +++ b/pkg/util/marshal/marshal_test.go @@ -71,8 +71,10 @@ var queryTests = []struct { "compressedBytes": 0, "decompressedBytes": 0, "decompressedLines": 0, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 0, "headChunkLines": 0, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 0 } @@ -91,8 +93,10 @@ var queryTests = []struct { "compressedBytes": 0, "decompressedBytes": 0, "decompressedLines": 0, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 0, "headChunkLines": 0, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 0 } @@ -147,6 +151,7 @@ var queryTests = []struct { "totalBytesProcessed": 0, "totalEntriesReturned": 0, "totalLinesProcessed": 0, + "totalNonIndexedLabelsBytesProcessed": 0, "totalPostFilterLines": 0 } } @@ -220,8 +225,10 @@ var queryTests = []struct { "compressedBytes": 0, "decompressedBytes": 0, "decompressedLines": 0, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 0, "headChunkLines": 0, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 0 } @@ -240,8 +247,10 @@ var queryTests = []struct { "compressedBytes": 0, "decompressedBytes": 0, "decompressedLines": 0, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 0, "headChunkLines": 0, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 0 } @@ -296,6 +305,7 @@ var queryTests = []struct { "totalBytesProcessed": 0, "totalEntriesReturned": 0, "totalLinesProcessed": 0, + "totalNonIndexedLabelsBytesProcessed": 0, "totalPostFilterLines": 0 } } @@ -390,8 +400,10 @@ var queryTests = []struct { "compressedBytes": 0, "decompressedBytes": 0, "decompressedLines": 0, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 0, "headChunkLines": 0, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 0 } @@ -410,8 +422,10 @@ var queryTests = []struct { "compressedBytes": 0, "decompressedBytes": 0, "decompressedLines": 0, + "decompressedNonIndexedLabelsBytes": 0, "headChunkBytes": 0, "headChunkLines": 0, + "headChunkNonIndexedLabelsBytes": 0, "postFilterLines": 0, "totalDuplicates": 0 } @@ -466,6 +480,7 @@ var queryTests = []struct { "totalBytesProcessed": 0, "totalEntriesReturned": 0, "totalLinesProcessed": 0, + "totalNonIndexedLabelsBytesProcessed": 0, "totalPostFilterLines": 0 } }