From 6fec92ab69abb8078a7258831918376350f13146 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Date: Thu, 28 May 2020 22:33:48 +0530 Subject: [PATCH] Upgrade Prometheus to current master (#2647) Signed-off-by: Ganesh Vernekar --- go.mod | 2 +- go.sum | 2 + .../prometheus/discovery/triton/triton.go | 87 +++++++++++++++++-- .../prometheus/prometheus/promql/engine.go | 4 + .../prometheus/prometheus/scrape/scrape.go | 6 +- .../prometheus/tsdb/chunks/head_chunks.go | 4 +- .../tsdb/chunks/head_chunks_other.go | 22 +++++ .../tsdb/chunks/head_chunks_windows.go | 20 +++++ vendor/modules.txt | 2 +- 9 files changed, 134 insertions(+), 15 deletions(-) create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks_other.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks_windows.go diff --git a/go.mod b/go.mod index 412e5fee4d..ca132cb79f 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/prometheus/client_golang v1.6.0 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.10.0 - github.com/prometheus/prometheus v1.8.2-0.20200522113006-f4dd45609a05 + github.com/prometheus/prometheus v1.8.2-0.20200528145419-83619aa9ac56 github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e github.com/spf13/afero v1.2.2 diff --git a/go.sum b/go.sum index f30a4ff239..a1ea0a2626 100644 --- a/go.sum +++ b/go.sum @@ -1005,6 +1005,8 @@ github.com/prometheus/prometheus v1.8.2-0.20200213233353-b90be6f32a33 h1:HBYrMJj github.com/prometheus/prometheus v1.8.2-0.20200213233353-b90be6f32a33/go.mod h1:fkIPPkuZnkXyopYHmXPxf9rgiPkVgZCN8w9o8+UgBlY= github.com/prometheus/prometheus v1.8.2-0.20200522113006-f4dd45609a05 h1:Tzn/Jcv/ORgKB59kGKNb95ThgrwuPBTgbdp09V+wM/Q= github.com/prometheus/prometheus v1.8.2-0.20200522113006-f4dd45609a05/go.mod h1:QKHYbx8sTY1fj75M+lL+LhzDSFM00+dOBlFn5wBi+14= +github.com/prometheus/prometheus v1.8.2-0.20200528145419-83619aa9ac56 h1:XtpfVkKbJQ+i+GguNtXalzgEd/vqADW9XBamdKdS+N8= +github.com/prometheus/prometheus v1.8.2-0.20200528145419-83619aa9ac56/go.mod h1:QKHYbx8sTY1fj75M+lL+LhzDSFM00+dOBlFn5wBi+14= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 h1:+kGqA4dNN5hn7WwvKdzHl0rdN5AEkbNZd0VjRltAiZg= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= diff --git a/vendor/github.com/prometheus/prometheus/discovery/triton/triton.go b/vendor/github.com/prometheus/prometheus/discovery/triton/triton.go index cef9e1c521..f00f5c8493 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/triton/triton.go +++ b/vendor/github.com/prometheus/prometheus/discovery/triton/triton.go @@ -46,6 +46,7 @@ const ( // DefaultSDConfig is the default Triton SD configuration. var DefaultSDConfig = SDConfig{ + Role: "container", Port: 9163, RefreshInterval: model.Duration(60 * time.Second), Version: 1, @@ -54,6 +55,7 @@ var DefaultSDConfig = SDConfig{ // SDConfig is the configuration for Triton based service discovery. type SDConfig struct { Account string `yaml:"account"` + Role string `yaml:"role"` DNSSuffix string `yaml:"dns_suffix"` Endpoint string `yaml:"endpoint"` Groups []string `yaml:"groups,omitempty"` @@ -71,6 +73,9 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { if err != nil { return err } + if c.Role != "container" && c.Role != "cn" { + return errors.New("triton SD configuration requires role to be 'container' or 'cn'") + } if c.Account == "" { return errors.New("triton SD configuration requires an account") } @@ -87,7 +92,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { } // DiscoveryResponse models a JSON response from the Triton discovery. -type discoveryResponse struct { +type DiscoveryResponse struct { Containers []struct { Groups []string `json:"groups"` ServerUUID string `json:"server_uuid"` @@ -98,6 +103,14 @@ type discoveryResponse struct { } `json:"containers"` } +// ComputeNodeDiscoveryResponse models a JSON response from the Triton discovery /gz/ endpoint. +type ComputeNodeDiscoveryResponse struct { + ComputeNodes []struct { + ServerUUID string `json:"server_uuid"` + ServerHostname string `json:"server_hostname"` + } `json:"cns"` +} + // Discovery periodically performs Triton-SD requests. It implements // the Discoverer interface. type Discovery struct { @@ -137,17 +150,34 @@ func New(logger log.Logger, conf *SDConfig) (*Discovery, error) { return d, nil } +// triton-cmon has two discovery endpoints: +// https://github.com/joyent/triton-cmon/blob/master/lib/endpoints/discover.js +// +// The default endpoint exposes "containers", otherwise called "virtual machines" in triton, +// which are (branded) zones running on the triton platform. +// +// The /gz/ endpoint exposes "compute nodes", also known as "servers" or "global zones", +// on which the "containers" are running. +// +// As triton is not internally consistent in using these names, +// the terms as used in triton-cmon are used here. + func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { - var endpoint = fmt.Sprintf("https://%s:%d/v%d/discover", d.sdConfig.Endpoint, d.sdConfig.Port, d.sdConfig.Version) + var endpointFormat string + switch d.sdConfig.Role { + case "container": + endpointFormat = "https://%s:%d/v%d/discover" + case "cn": + endpointFormat = "https://%s:%d/v%d/gz/discover" + default: + return nil, errors.New(fmt.Sprintf("unknown role '%s' in configuration", d.sdConfig.Role)) + } + var endpoint = fmt.Sprintf(endpointFormat, d.sdConfig.Endpoint, d.sdConfig.Port, d.sdConfig.Version) if len(d.sdConfig.Groups) > 0 { groups := url.QueryEscape(strings.Join(d.sdConfig.Groups, ",")) endpoint = fmt.Sprintf("%s?groups=%s", endpoint, groups) } - tg := &targetgroup.Group{ - Source: endpoint, - } - req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -168,8 +198,24 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { return nil, errors.Wrap(err, "an error occurred when reading the response body") } - dr := discoveryResponse{} - err = json.Unmarshal(data, &dr) + // The JSON response body is different so it needs to be processed/mapped separately. + switch d.sdConfig.Role { + case "container": + return d.processContainerResponse(data, endpoint) + case "cn": + return d.processComputeNodeResponse(data, endpoint) + default: + return nil, errors.New(fmt.Sprintf("unknown role '%s' in configuration", d.sdConfig.Role)) + } +} + +func (d *Discovery) processContainerResponse(data []byte, endpoint string) ([]*targetgroup.Group, error) { + tg := &targetgroup.Group{ + Source: endpoint, + } + + dr := DiscoveryResponse{} + err := json.Unmarshal(data, &dr) if err != nil { return nil, errors.Wrap(err, "an error occurred unmarshaling the discovery response json") } @@ -195,3 +241,28 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { return []*targetgroup.Group{tg}, nil } + +func (d *Discovery) processComputeNodeResponse(data []byte, endpoint string) ([]*targetgroup.Group, error) { + tg := &targetgroup.Group{ + Source: endpoint, + } + + dr := ComputeNodeDiscoveryResponse{} + err := json.Unmarshal(data, &dr) + if err != nil { + return nil, errors.Wrap(err, "an error occurred unmarshaling the compute node discovery response json") + } + + for _, cn := range dr.ComputeNodes { + labels := model.LabelSet{ + tritonLabelMachineID: model.LabelValue(cn.ServerUUID), + tritonLabelMachineAlias: model.LabelValue(cn.ServerHostname), + } + addr := fmt.Sprintf("%s.%s:%d", cn.ServerUUID, d.sdConfig.DNSSuffix, d.sdConfig.Port) + labels[model.AddressLabel] = model.LabelValue(addr) + + tg.Targets = append(tg.Targets, labels) + } + + return []*targetgroup.Group{tg}, nil +} diff --git a/vendor/github.com/prometheus/prometheus/promql/engine.go b/vendor/github.com/prometheus/prometheus/promql/engine.go index c3f0002361..c7f02b7ae7 100644 --- a/vendor/github.com/prometheus/prometheus/promql/engine.go +++ b/vendor/github.com/prometheus/prometheus/promql/engine.go @@ -1120,6 +1120,7 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { // Process all the calls for one time series at a time. it := storage.NewBuffer(selRange) for i, s := range selVS.Series { + ev.currentSamples -= len(points) points = points[:0] it.Reset(s.Iterator()) ss := Series{ @@ -1170,6 +1171,7 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { } } + ev.currentSamples -= len(points) putPointSlice(points) // The absent_over_time function returns 0 or 1 series. So far, the matrix @@ -1480,11 +1482,13 @@ func (ev *evaluator) matrixIterSlice(it *storage.BufferedSeriesIterator, mint, m var drop int for drop = 0; out[drop].T < mint; drop++ { } + ev.currentSamples -= drop copy(out, out[drop:]) out = out[:len(out)-drop] // Only append points with timestamps after the last timestamp we have. mint = out[len(out)-1].T + 1 } else { + ev.currentSamples -= len(out) out = out[:0] } diff --git a/vendor/github.com/prometheus/prometheus/scrape/scrape.go b/vendor/github.com/prometheus/prometheus/scrape/scrape.go index f63711e3fa..d1987e97c6 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/scrape.go +++ b/vendor/github.com/prometheus/prometheus/scrape/scrape.go @@ -1140,7 +1140,7 @@ loop: if ok { err = app.AddFast(ce.ref, t, v) - sampleAdded, err = sl.checkAddError(ce, met, tp, err, &sampleLimitErr, appErrs) + sampleAdded, err = sl.checkAddError(ce, met, tp, err, &sampleLimitErr, &appErrs) // In theory this should never happen. if err == storage.ErrNotFound { ok = false @@ -1169,7 +1169,7 @@ loop: var ref uint64 ref, err = app.Add(lset, t, v) - sampleAdded, err = sl.checkAddError(nil, met, tp, err, &sampleLimitErr, appErrs) + sampleAdded, err = sl.checkAddError(nil, met, tp, err, &sampleLimitErr, &appErrs) if err != nil { if err != storage.ErrNotFound { level.Debug(sl.l).Log("msg", "Unexpected error", "series", string(met), "err", err) @@ -1231,7 +1231,7 @@ func yoloString(b []byte) string { // Adds samples to the appender, checking the error, and then returns the # of samples added, // whether the caller should continue to process more samples, and any sample limit errors. -func (sl *scrapeLoop) checkAddError(ce *cacheEntry, met []byte, tp *int64, err error, sampleLimitErr *error, appErrs appendErrors) (bool, error) { +func (sl *scrapeLoop) checkAddError(ce *cacheEntry, met []byte, tp *int64, err error, sampleLimitErr *error, appErrs *appendErrors) (bool, error) { switch errors.Cause(err) { case nil: if tp == nil && ce != nil { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go b/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go index 8363c67dc2..9d39fc986c 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go @@ -188,7 +188,7 @@ func (cdm *ChunkDiskMapper) openMMapFiles() (returnErr error) { for seq, fn := range files { f, err := fileutil.OpenMmapFile(fn) if err != nil { - return errors.Wrap(err, "mmap files") + return errors.Wrapf(err, "mmap files, file: %s", fn) } cdm.closers[seq] = f cdm.mmappedChunkFiles[seq] = &mmappedChunkFile{byteSlice: realByteSlice(f.Bytes())} @@ -337,7 +337,7 @@ func (cdm *ChunkDiskMapper) cut(mint int64) (returnErr error) { return err } - n, newFile, seq, err := cutSegmentFile(cdm.dir, MagicHeadChunks, headChunksFormatV1, int64(MaxHeadChunkFileSize)) + n, newFile, seq, err := cutSegmentFile(cdm.dir, MagicHeadChunks, headChunksFormatV1, HeadChunkFilePreallocationSize) if err != nil { return err } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks_other.go b/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks_other.go new file mode 100644 index 0000000000..a1de873707 --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks_other.go @@ -0,0 +1,22 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package chunks + +var ( + // HeadChunkFilePreallocationSize is the size to which the m-map file should be preallocated when a new file is cut. + // Windows needs pre-allocations while the other OS does not. + HeadChunkFilePreallocationSize int64 +) diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks_windows.go b/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks_windows.go new file mode 100644 index 0000000000..b772b64b46 --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks_windows.go @@ -0,0 +1,20 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chunks + +var ( + // HeadChunkFilePreallocationSize is the size to which the m-map file should be preallocated when a new file is cut. + // Windows needs pre-allocation to m-map the file. + HeadChunkFilePreallocationSize int64 = MaxHeadChunkFileSize +) diff --git a/vendor/modules.txt b/vendor/modules.txt index 35b958e82a..e7dfa3b19d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -442,7 +442,7 @@ github.com/prometheus/node_exporter/https github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/prometheus/prometheus v1.8.2-0.20200522113006-f4dd45609a05 +# github.com/prometheus/prometheus v1.8.2-0.20200528145419-83619aa9ac56 github.com/prometheus/prometheus/config github.com/prometheus/prometheus/discovery github.com/prometheus/prometheus/discovery/azure