From 57e3f1d0556ef06c834b24764cd0f22d7a4599f1 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Tue, 10 Sep 2019 21:46:18 +0200 Subject: [PATCH 01/18] chore(vendoring): ghodss/yaml, kelseyhightower/envconfig --- Gopkg.lock | 18 + vendor/github.com/ghodss/yaml/.gitignore | 20 + vendor/github.com/ghodss/yaml/.travis.yml | 7 + vendor/github.com/ghodss/yaml/LICENSE | 50 ++ vendor/github.com/ghodss/yaml/README.md | 121 +++++ vendor/github.com/ghodss/yaml/fields.go | 501 ++++++++++++++++++ vendor/github.com/ghodss/yaml/yaml.go | 277 ++++++++++ .../kelseyhightower/envconfig/.travis.yml | 13 + .../kelseyhightower/envconfig/LICENSE | 19 + .../kelseyhightower/envconfig/MAINTAINERS | 2 + .../kelseyhightower/envconfig/README.md | 192 +++++++ .../kelseyhightower/envconfig/doc.go | 8 + .../kelseyhightower/envconfig/env_os.go | 7 + .../kelseyhightower/envconfig/env_syscall.go | 7 + .../kelseyhightower/envconfig/envconfig.go | 382 +++++++++++++ .../kelseyhightower/envconfig/go.mod | 1 + .../kelseyhightower/envconfig/usage.go | 164 ++++++ 17 files changed, 1789 insertions(+) create mode 100644 vendor/github.com/ghodss/yaml/.gitignore create mode 100644 vendor/github.com/ghodss/yaml/.travis.yml create mode 100644 vendor/github.com/ghodss/yaml/LICENSE create mode 100644 vendor/github.com/ghodss/yaml/README.md create mode 100644 vendor/github.com/ghodss/yaml/fields.go create mode 100644 vendor/github.com/ghodss/yaml/yaml.go create mode 100644 vendor/github.com/kelseyhightower/envconfig/.travis.yml create mode 100644 vendor/github.com/kelseyhightower/envconfig/LICENSE create mode 100644 vendor/github.com/kelseyhightower/envconfig/MAINTAINERS create mode 100644 vendor/github.com/kelseyhightower/envconfig/README.md create mode 100644 vendor/github.com/kelseyhightower/envconfig/doc.go create mode 100644 vendor/github.com/kelseyhightower/envconfig/env_os.go create mode 100644 vendor/github.com/kelseyhightower/envconfig/env_syscall.go create mode 100644 vendor/github.com/kelseyhightower/envconfig/envconfig.go create mode 100644 vendor/github.com/kelseyhightower/envconfig/go.mod create mode 100644 vendor/github.com/kelseyhightower/envconfig/usage.go diff --git a/Gopkg.lock b/Gopkg.lock index 5b4b2775e3877..98587ca01d2db 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -423,6 +423,14 @@ pruneopts = "UT" revision = "b364c791f57ac8f224c2f42ad599afa45ca0cfda" +[[projects]] + digest = "1:2cd7915ab26ede7d95b8749e6b1f933f1c6d5398030684e6505940a10f31cfda" + name = "github.com/ghodss/yaml" + packages = ["."] + pruneopts = "UT" + revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" + version = "v1.0.0" + [[projects]] digest = "1:ce058ca1b1787a997e578c4344f9650becb5086a3512332cbedffca9950d77a1" name = "github.com/go-kit/kit" @@ -733,6 +741,14 @@ revision = "1624edc4454b8682399def8740d46db5e4362ba4" version = "v1.1.5" +[[projects]] + digest = "1:fd9bea48bbc5bba66d9891c72af7255fbebecdff845c37c679406174ece5ca1b" + name = "github.com/kelseyhightower/envconfig" + packages = ["."] + pruneopts = "UT" + revision = "0b417c4ec4a8a82eecc22a1459a504aa55163d61" + version = "v1.4.0" + [[projects]] digest = "1:9b67e6f39db9eb120a6c5977e4b29f33d7e01970777e77152ed79395f7559291" name = "github.com/klauspost/compress" @@ -1841,6 +1857,7 @@ "github.com/docker/docker/pkg/ioutils", "github.com/docker/go-plugins-helpers/sdk", "github.com/fatih/color", + "github.com/ghodss/yaml", "github.com/go-kit/kit/log", "github.com/go-kit/kit/log/level", "github.com/gogo/protobuf/gogoproto", @@ -1853,6 +1870,7 @@ "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc", "github.com/hpcloud/tail", "github.com/jmespath/go-jmespath", + "github.com/kelseyhightower/envconfig", "github.com/klauspost/compress/gzip", "github.com/mitchellh/mapstructure", "github.com/opentracing/opentracing-go", diff --git a/vendor/github.com/ghodss/yaml/.gitignore b/vendor/github.com/ghodss/yaml/.gitignore new file mode 100644 index 0000000000000..e256a31e00a52 --- /dev/null +++ b/vendor/github.com/ghodss/yaml/.gitignore @@ -0,0 +1,20 @@ +# OSX leaves these everywhere on SMB shares +._* + +# Eclipse files +.classpath +.project +.settings/** + +# Emacs save files +*~ + +# Vim-related files +[._]*.s[a-w][a-z] +[._]s[a-w][a-z] +*.un~ +Session.vim +.netrwhist + +# Go test binaries +*.test diff --git a/vendor/github.com/ghodss/yaml/.travis.yml b/vendor/github.com/ghodss/yaml/.travis.yml new file mode 100644 index 0000000000000..0e9d6edc010a6 --- /dev/null +++ b/vendor/github.com/ghodss/yaml/.travis.yml @@ -0,0 +1,7 @@ +language: go +go: + - 1.3 + - 1.4 +script: + - go test + - go build diff --git a/vendor/github.com/ghodss/yaml/LICENSE b/vendor/github.com/ghodss/yaml/LICENSE new file mode 100644 index 0000000000000..7805d36de7305 --- /dev/null +++ b/vendor/github.com/ghodss/yaml/LICENSE @@ -0,0 +1,50 @@ +The MIT License (MIT) + +Copyright (c) 2014 Sam Ghods + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/ghodss/yaml/README.md b/vendor/github.com/ghodss/yaml/README.md new file mode 100644 index 0000000000000..0200f75b4d126 --- /dev/null +++ b/vendor/github.com/ghodss/yaml/README.md @@ -0,0 +1,121 @@ +# YAML marshaling and unmarshaling support for Go + +[![Build Status](https://travis-ci.org/ghodss/yaml.svg)](https://travis-ci.org/ghodss/yaml) + +## Introduction + +A wrapper around [go-yaml](https://github.com/go-yaml/yaml) designed to enable a better way of handling YAML when marshaling to and from structs. + +In short, this library first converts YAML to JSON using go-yaml and then uses `json.Marshal` and `json.Unmarshal` to convert to or from the struct. This means that it effectively reuses the JSON struct tags as well as the custom JSON methods `MarshalJSON` and `UnmarshalJSON` unlike go-yaml. For a detailed overview of the rationale behind this method, [see this blog post](http://ghodss.com/2014/the-right-way-to-handle-yaml-in-golang/). + +## Compatibility + +This package uses [go-yaml](https://github.com/go-yaml/yaml) and therefore supports [everything go-yaml supports](https://github.com/go-yaml/yaml#compatibility). + +## Caveats + +**Caveat #1:** When using `yaml.Marshal` and `yaml.Unmarshal`, binary data should NOT be preceded with the `!!binary` YAML tag. If you do, go-yaml will convert the binary data from base64 to native binary data, which is not compatible with JSON. You can still use binary in your YAML files though - just store them without the `!!binary` tag and decode the base64 in your code (e.g. in the custom JSON methods `MarshalJSON` and `UnmarshalJSON`). This also has the benefit that your YAML and your JSON binary data will be decoded exactly the same way. As an example: + +``` +BAD: + exampleKey: !!binary gIGC + +GOOD: + exampleKey: gIGC +... and decode the base64 data in your code. +``` + +**Caveat #2:** When using `YAMLToJSON` directly, maps with keys that are maps will result in an error since this is not supported by JSON. This error will occur in `Unmarshal` as well since you can't unmarshal map keys anyways since struct fields can't be keys. + +## Installation and usage + +To install, run: + +``` +$ go get github.com/ghodss/yaml +``` + +And import using: + +``` +import "github.com/ghodss/yaml" +``` + +Usage is very similar to the JSON library: + +```go +package main + +import ( + "fmt" + + "github.com/ghodss/yaml" +) + +type Person struct { + Name string `json:"name"` // Affects YAML field names too. + Age int `json:"age"` +} + +func main() { + // Marshal a Person struct to YAML. + p := Person{"John", 30} + y, err := yaml.Marshal(p) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + fmt.Println(string(y)) + /* Output: + age: 30 + name: John + */ + + // Unmarshal the YAML back into a Person struct. + var p2 Person + err = yaml.Unmarshal(y, &p2) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + fmt.Println(p2) + /* Output: + {John 30} + */ +} +``` + +`yaml.YAMLToJSON` and `yaml.JSONToYAML` methods are also available: + +```go +package main + +import ( + "fmt" + + "github.com/ghodss/yaml" +) + +func main() { + j := []byte(`{"name": "John", "age": 30}`) + y, err := yaml.JSONToYAML(j) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + fmt.Println(string(y)) + /* Output: + name: John + age: 30 + */ + j2, err := yaml.YAMLToJSON(y) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + fmt.Println(string(j2)) + /* Output: + {"age":30,"name":"John"} + */ +} +``` diff --git a/vendor/github.com/ghodss/yaml/fields.go b/vendor/github.com/ghodss/yaml/fields.go new file mode 100644 index 0000000000000..58600740266c6 --- /dev/null +++ b/vendor/github.com/ghodss/yaml/fields.go @@ -0,0 +1,501 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package yaml + +import ( + "bytes" + "encoding" + "encoding/json" + "reflect" + "sort" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +// indirect walks down v allocating pointers as needed, +// until it gets to a non-pointer. +// if it encounters an Unmarshaler, indirect stops and returns that. +// if decodingNull is true, indirect stops at the last pointer so it can be set to nil. +func indirect(v reflect.Value, decodingNull bool) (json.Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { + // If v is a named type and is addressable, + // start with its address, so that if the type has pointer methods, + // we find them. + if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { + v = v.Addr() + } + for { + // Load value from interface, but only if the result will be + // usefully addressable. + if v.Kind() == reflect.Interface && !v.IsNil() { + e := v.Elem() + if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { + v = e + continue + } + } + + if v.Kind() != reflect.Ptr { + break + } + + if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() { + break + } + if v.IsNil() { + if v.CanSet() { + v.Set(reflect.New(v.Type().Elem())) + } else { + v = reflect.New(v.Type().Elem()) + } + } + if v.Type().NumMethod() > 0 { + if u, ok := v.Interface().(json.Unmarshaler); ok { + return u, nil, reflect.Value{} + } + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + return nil, u, reflect.Value{} + } + } + v = v.Elem() + } + return nil, nil, v +} + +// A field represents a single field found in a struct. +type field struct { + name string + nameBytes []byte // []byte(name) + equalFold func(s, t []byte) bool // bytes.EqualFold or equivalent + + tag bool + index []int + typ reflect.Type + omitEmpty bool + quoted bool +} + +func fillField(f field) field { + f.nameBytes = []byte(f.name) + f.equalFold = foldFunc(f.nameBytes) + return f +} + +// byName sorts field by name, breaking ties with depth, +// then breaking ties with "name came from json tag", then +// breaking ties with index sequence. +type byName []field + +func (x byName) Len() int { return len(x) } + +func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byName) Less(i, j int) bool { + if x[i].name != x[j].name { + return x[i].name < x[j].name + } + if len(x[i].index) != len(x[j].index) { + return len(x[i].index) < len(x[j].index) + } + if x[i].tag != x[j].tag { + return x[i].tag + } + return byIndex(x).Less(i, j) +} + +// byIndex sorts field by index sequence. +type byIndex []field + +func (x byIndex) Len() int { return len(x) } + +func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byIndex) Less(i, j int) bool { + for k, xik := range x[i].index { + if k >= len(x[j].index) { + return false + } + if xik != x[j].index[k] { + return xik < x[j].index[k] + } + } + return len(x[i].index) < len(x[j].index) +} + +// typeFields returns a list of fields that JSON should recognize for the given type. +// The algorithm is breadth-first search over the set of structs to include - the top struct +// and then any reachable anonymous structs. +func typeFields(t reflect.Type) []field { + // Anonymous fields to explore at the current level and the next. + current := []field{} + next := []field{{typ: t}} + + // Count of queued names for current level and the next. + count := map[reflect.Type]int{} + nextCount := map[reflect.Type]int{} + + // Types already visited at an earlier level. + visited := map[reflect.Type]bool{} + + // Fields found. + var fields []field + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, map[reflect.Type]int{} + + for _, f := range current { + if visited[f.typ] { + continue + } + visited[f.typ] = true + + // Scan f.typ for fields to include. + for i := 0; i < f.typ.NumField(); i++ { + sf := f.typ.Field(i) + if sf.PkgPath != "" { // unexported + continue + } + tag := sf.Tag.Get("json") + if tag == "-" { + continue + } + name, opts := parseTag(tag) + if !isValidTag(name) { + name = "" + } + index := make([]int, len(f.index)+1) + copy(index, f.index) + index[len(f.index)] = i + + ft := sf.Type + if ft.Name() == "" && ft.Kind() == reflect.Ptr { + // Follow pointer. + ft = ft.Elem() + } + + // Record found field and index sequence. + if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := name != "" + if name == "" { + name = sf.Name + } + fields = append(fields, fillField(field{ + name: name, + tag: tagged, + index: index, + typ: ft, + omitEmpty: opts.Contains("omitempty"), + quoted: opts.Contains("string"), + })) + if count[f.typ] > 1 { + // If there were multiple instances, add a second, + // so that the annihilation code will see a duplicate. + // It only cares about the distinction between 1 or 2, + // so don't bother generating any more copies. + fields = append(fields, fields[len(fields)-1]) + } + continue + } + + // Record new anonymous struct to explore in next round. + nextCount[ft]++ + if nextCount[ft] == 1 { + next = append(next, fillField(field{name: ft.Name(), index: index, typ: ft})) + } + } + } + } + + sort.Sort(byName(fields)) + + // Delete all fields that are hidden by the Go rules for embedded fields, + // except that fields with JSON tags are promoted. + + // The fields are sorted in primary order of name, secondary order + // of field index length. Loop over names; for each name, delete + // hidden fields by choosing the one dominant field that survives. + out := fields[:0] + for advance, i := 0, 0; i < len(fields); i += advance { + // One iteration per name. + // Find the sequence of fields with the name of this first field. + fi := fields[i] + name := fi.name + for advance = 1; i+advance < len(fields); advance++ { + fj := fields[i+advance] + if fj.name != name { + break + } + } + if advance == 1 { // Only one field with this name + out = append(out, fi) + continue + } + dominant, ok := dominantField(fields[i : i+advance]) + if ok { + out = append(out, dominant) + } + } + + fields = out + sort.Sort(byIndex(fields)) + + return fields +} + +// dominantField looks through the fields, all of which are known to +// have the same name, to find the single field that dominates the +// others using Go's embedding rules, modified by the presence of +// JSON tags. If there are multiple top-level fields, the boolean +// will be false: This condition is an error in Go and we skip all +// the fields. +func dominantField(fields []field) (field, bool) { + // The fields are sorted in increasing index-length order. The winner + // must therefore be one with the shortest index length. Drop all + // longer entries, which is easy: just truncate the slice. + length := len(fields[0].index) + tagged := -1 // Index of first tagged field. + for i, f := range fields { + if len(f.index) > length { + fields = fields[:i] + break + } + if f.tag { + if tagged >= 0 { + // Multiple tagged fields at the same level: conflict. + // Return no field. + return field{}, false + } + tagged = i + } + } + if tagged >= 0 { + return fields[tagged], true + } + // All remaining fields have the same length. If there's more than one, + // we have a conflict (two fields named "X" at the same level) and we + // return no field. + if len(fields) > 1 { + return field{}, false + } + return fields[0], true +} + +var fieldCache struct { + sync.RWMutex + m map[reflect.Type][]field +} + +// cachedTypeFields is like typeFields but uses a cache to avoid repeated work. +func cachedTypeFields(t reflect.Type) []field { + fieldCache.RLock() + f := fieldCache.m[t] + fieldCache.RUnlock() + if f != nil { + return f + } + + // Compute fields without lock. + // Might duplicate effort but won't hold other computations back. + f = typeFields(t) + if f == nil { + f = []field{} + } + + fieldCache.Lock() + if fieldCache.m == nil { + fieldCache.m = map[reflect.Type][]field{} + } + fieldCache.m[t] = f + fieldCache.Unlock() + return f +} + +func isValidTag(s string) bool { + if s == "" { + return false + } + for _, c := range s { + switch { + case strings.ContainsRune("!#$%&()*+-./:<=>?@[]^_{|}~ ", c): + // Backslash and quote chars are reserved, but + // otherwise any punctuation chars are allowed + // in a tag name. + default: + if !unicode.IsLetter(c) && !unicode.IsDigit(c) { + return false + } + } + } + return true +} + +const ( + caseMask = ^byte(0x20) // Mask to ignore case in ASCII. + kelvin = '\u212a' + smallLongEss = '\u017f' +) + +// foldFunc returns one of four different case folding equivalence +// functions, from most general (and slow) to fastest: +// +// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8 +// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S') +// 3) asciiEqualFold, no special, but includes non-letters (including _) +// 4) simpleLetterEqualFold, no specials, no non-letters. +// +// The letters S and K are special because they map to 3 runes, not just 2: +// * S maps to s and to U+017F 'ſ' Latin small letter long s +// * k maps to K and to U+212A 'K' Kelvin sign +// See http://play.golang.org/p/tTxjOc0OGo +// +// The returned function is specialized for matching against s and +// should only be given s. It's not curried for performance reasons. +func foldFunc(s []byte) func(s, t []byte) bool { + nonLetter := false + special := false // special letter + for _, b := range s { + if b >= utf8.RuneSelf { + return bytes.EqualFold + } + upper := b & caseMask + if upper < 'A' || upper > 'Z' { + nonLetter = true + } else if upper == 'K' || upper == 'S' { + // See above for why these letters are special. + special = true + } + } + if special { + return equalFoldRight + } + if nonLetter { + return asciiEqualFold + } + return simpleLetterEqualFold +} + +// equalFoldRight is a specialization of bytes.EqualFold when s is +// known to be all ASCII (including punctuation), but contains an 's', +// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t. +// See comments on foldFunc. +func equalFoldRight(s, t []byte) bool { + for _, sb := range s { + if len(t) == 0 { + return false + } + tb := t[0] + if tb < utf8.RuneSelf { + if sb != tb { + sbUpper := sb & caseMask + if 'A' <= sbUpper && sbUpper <= 'Z' { + if sbUpper != tb&caseMask { + return false + } + } else { + return false + } + } + t = t[1:] + continue + } + // sb is ASCII and t is not. t must be either kelvin + // sign or long s; sb must be s, S, k, or K. + tr, size := utf8.DecodeRune(t) + switch sb { + case 's', 'S': + if tr != smallLongEss { + return false + } + case 'k', 'K': + if tr != kelvin { + return false + } + default: + return false + } + t = t[size:] + + } + if len(t) > 0 { + return false + } + return true +} + +// asciiEqualFold is a specialization of bytes.EqualFold for use when +// s is all ASCII (but may contain non-letters) and contains no +// special-folding letters. +// See comments on foldFunc. +func asciiEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, sb := range s { + tb := t[i] + if sb == tb { + continue + } + if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') { + if sb&caseMask != tb&caseMask { + return false + } + } else { + return false + } + } + return true +} + +// simpleLetterEqualFold is a specialization of bytes.EqualFold for +// use when s is all ASCII letters (no underscores, etc) and also +// doesn't contain 'k', 'K', 's', or 'S'. +// See comments on foldFunc. +func simpleLetterEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, b := range s { + if b&caseMask != t[i]&caseMask { + return false + } + } + return true +} + +// tagOptions is the string following a comma in a struct field's "json" +// tag, or the empty string. It does not include the leading comma. +type tagOptions string + +// parseTag splits a struct field's json tag into its name and +// comma-separated options. +func parseTag(tag string) (string, tagOptions) { + if idx := strings.Index(tag, ","); idx != -1 { + return tag[:idx], tagOptions(tag[idx+1:]) + } + return tag, tagOptions("") +} + +// Contains reports whether a comma-separated list of options +// contains a particular substr flag. substr must be surrounded by a +// string boundary or commas. +func (o tagOptions) Contains(optionName string) bool { + if len(o) == 0 { + return false + } + s := string(o) + for s != "" { + var next string + i := strings.Index(s, ",") + if i >= 0 { + s, next = s[:i], s[i+1:] + } + if s == optionName { + return true + } + s = next + } + return false +} diff --git a/vendor/github.com/ghodss/yaml/yaml.go b/vendor/github.com/ghodss/yaml/yaml.go new file mode 100644 index 0000000000000..4fb4054a8b747 --- /dev/null +++ b/vendor/github.com/ghodss/yaml/yaml.go @@ -0,0 +1,277 @@ +package yaml + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strconv" + + "gopkg.in/yaml.v2" +) + +// Marshals the object into JSON then converts JSON to YAML and returns the +// YAML. +func Marshal(o interface{}) ([]byte, error) { + j, err := json.Marshal(o) + if err != nil { + return nil, fmt.Errorf("error marshaling into JSON: %v", err) + } + + y, err := JSONToYAML(j) + if err != nil { + return nil, fmt.Errorf("error converting JSON to YAML: %v", err) + } + + return y, nil +} + +// Converts YAML to JSON then uses JSON to unmarshal into an object. +func Unmarshal(y []byte, o interface{}) error { + vo := reflect.ValueOf(o) + j, err := yamlToJSON(y, &vo) + if err != nil { + return fmt.Errorf("error converting YAML to JSON: %v", err) + } + + err = json.Unmarshal(j, o) + if err != nil { + return fmt.Errorf("error unmarshaling JSON: %v", err) + } + + return nil +} + +// Convert JSON to YAML. +func JSONToYAML(j []byte) ([]byte, error) { + // Convert the JSON to an object. + var jsonObj interface{} + // We are using yaml.Unmarshal here (instead of json.Unmarshal) because the + // Go JSON library doesn't try to pick the right number type (int, float, + // etc.) when unmarshalling to interface{}, it just picks float64 + // universally. go-yaml does go through the effort of picking the right + // number type, so we can preserve number type throughout this process. + err := yaml.Unmarshal(j, &jsonObj) + if err != nil { + return nil, err + } + + // Marshal this object into YAML. + return yaml.Marshal(jsonObj) +} + +// Convert YAML to JSON. Since JSON is a subset of YAML, passing JSON through +// this method should be a no-op. +// +// Things YAML can do that are not supported by JSON: +// * In YAML you can have binary and null keys in your maps. These are invalid +// in JSON. (int and float keys are converted to strings.) +// * Binary data in YAML with the !!binary tag is not supported. If you want to +// use binary data with this library, encode the data as base64 as usual but do +// not use the !!binary tag in your YAML. This will ensure the original base64 +// encoded data makes it all the way through to the JSON. +func YAMLToJSON(y []byte) ([]byte, error) { + return yamlToJSON(y, nil) +} + +func yamlToJSON(y []byte, jsonTarget *reflect.Value) ([]byte, error) { + // Convert the YAML to an object. + var yamlObj interface{} + err := yaml.Unmarshal(y, &yamlObj) + if err != nil { + return nil, err + } + + // YAML objects are not completely compatible with JSON objects (e.g. you + // can have non-string keys in YAML). So, convert the YAML-compatible object + // to a JSON-compatible object, failing with an error if irrecoverable + // incompatibilties happen along the way. + jsonObj, err := convertToJSONableObject(yamlObj, jsonTarget) + if err != nil { + return nil, err + } + + // Convert this object to JSON and return the data. + return json.Marshal(jsonObj) +} + +func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (interface{}, error) { + var err error + + // Resolve jsonTarget to a concrete value (i.e. not a pointer or an + // interface). We pass decodingNull as false because we're not actually + // decoding into the value, we're just checking if the ultimate target is a + // string. + if jsonTarget != nil { + ju, tu, pv := indirect(*jsonTarget, false) + // We have a JSON or Text Umarshaler at this level, so we can't be trying + // to decode into a string. + if ju != nil || tu != nil { + jsonTarget = nil + } else { + jsonTarget = &pv + } + } + + // If yamlObj is a number or a boolean, check if jsonTarget is a string - + // if so, coerce. Else return normal. + // If yamlObj is a map or array, find the field that each key is + // unmarshaling to, and when you recurse pass the reflect.Value for that + // field back into this function. + switch typedYAMLObj := yamlObj.(type) { + case map[interface{}]interface{}: + // JSON does not support arbitrary keys in a map, so we must convert + // these keys to strings. + // + // From my reading of go-yaml v2 (specifically the resolve function), + // keys can only have the types string, int, int64, float64, binary + // (unsupported), or null (unsupported). + strMap := make(map[string]interface{}) + for k, v := range typedYAMLObj { + // Resolve the key to a string first. + var keyString string + switch typedKey := k.(type) { + case string: + keyString = typedKey + case int: + keyString = strconv.Itoa(typedKey) + case int64: + // go-yaml will only return an int64 as a key if the system + // architecture is 32-bit and the key's value is between 32-bit + // and 64-bit. Otherwise the key type will simply be int. + keyString = strconv.FormatInt(typedKey, 10) + case float64: + // Stolen from go-yaml to use the same conversion to string as + // the go-yaml library uses to convert float to string when + // Marshaling. + s := strconv.FormatFloat(typedKey, 'g', -1, 32) + switch s { + case "+Inf": + s = ".inf" + case "-Inf": + s = "-.inf" + case "NaN": + s = ".nan" + } + keyString = s + case bool: + if typedKey { + keyString = "true" + } else { + keyString = "false" + } + default: + return nil, fmt.Errorf("Unsupported map key of type: %s, key: %+#v, value: %+#v", + reflect.TypeOf(k), k, v) + } + + // jsonTarget should be a struct or a map. If it's a struct, find + // the field it's going to map to and pass its reflect.Value. If + // it's a map, find the element type of the map and pass the + // reflect.Value created from that type. If it's neither, just pass + // nil - JSON conversion will error for us if it's a real issue. + if jsonTarget != nil { + t := *jsonTarget + if t.Kind() == reflect.Struct { + keyBytes := []byte(keyString) + // Find the field that the JSON library would use. + var f *field + fields := cachedTypeFields(t.Type()) + for i := range fields { + ff := &fields[i] + if bytes.Equal(ff.nameBytes, keyBytes) { + f = ff + break + } + // Do case-insensitive comparison. + if f == nil && ff.equalFold(ff.nameBytes, keyBytes) { + f = ff + } + } + if f != nil { + // Find the reflect.Value of the most preferential + // struct field. + jtf := t.Field(f.index[0]) + strMap[keyString], err = convertToJSONableObject(v, &jtf) + if err != nil { + return nil, err + } + continue + } + } else if t.Kind() == reflect.Map { + // Create a zero value of the map's element type to use as + // the JSON target. + jtv := reflect.Zero(t.Type().Elem()) + strMap[keyString], err = convertToJSONableObject(v, &jtv) + if err != nil { + return nil, err + } + continue + } + } + strMap[keyString], err = convertToJSONableObject(v, nil) + if err != nil { + return nil, err + } + } + return strMap, nil + case []interface{}: + // We need to recurse into arrays in case there are any + // map[interface{}]interface{}'s inside and to convert any + // numbers to strings. + + // If jsonTarget is a slice (which it really should be), find the + // thing it's going to map to. If it's not a slice, just pass nil + // - JSON conversion will error for us if it's a real issue. + var jsonSliceElemValue *reflect.Value + if jsonTarget != nil { + t := *jsonTarget + if t.Kind() == reflect.Slice { + // By default slices point to nil, but we need a reflect.Value + // pointing to a value of the slice type, so we create one here. + ev := reflect.Indirect(reflect.New(t.Type().Elem())) + jsonSliceElemValue = &ev + } + } + + // Make and use a new array. + arr := make([]interface{}, len(typedYAMLObj)) + for i, v := range typedYAMLObj { + arr[i], err = convertToJSONableObject(v, jsonSliceElemValue) + if err != nil { + return nil, err + } + } + return arr, nil + default: + // If the target type is a string and the YAML type is a number, + // convert the YAML type to a string. + if jsonTarget != nil && (*jsonTarget).Kind() == reflect.String { + // Based on my reading of go-yaml, it may return int, int64, + // float64, or uint64. + var s string + switch typedVal := typedYAMLObj.(type) { + case int: + s = strconv.FormatInt(int64(typedVal), 10) + case int64: + s = strconv.FormatInt(typedVal, 10) + case float64: + s = strconv.FormatFloat(typedVal, 'g', -1, 32) + case uint64: + s = strconv.FormatUint(typedVal, 10) + case bool: + if typedVal { + s = "true" + } else { + s = "false" + } + } + if len(s) > 0 { + yamlObj = interface{}(s) + } + } + return yamlObj, nil + } + + return nil, nil +} diff --git a/vendor/github.com/kelseyhightower/envconfig/.travis.yml b/vendor/github.com/kelseyhightower/envconfig/.travis.yml new file mode 100644 index 0000000000000..04b97aed6164e --- /dev/null +++ b/vendor/github.com/kelseyhightower/envconfig/.travis.yml @@ -0,0 +1,13 @@ +language: go + +go: + - 1.4.x + - 1.5.x + - 1.6.x + - 1.7.x + - 1.8.x + - 1.9.x + - 1.10.x + - 1.11.x + - 1.12.x + - tip diff --git a/vendor/github.com/kelseyhightower/envconfig/LICENSE b/vendor/github.com/kelseyhightower/envconfig/LICENSE new file mode 100644 index 0000000000000..4bfa7a84d818d --- /dev/null +++ b/vendor/github.com/kelseyhightower/envconfig/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2013 Kelsey Hightower + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/kelseyhightower/envconfig/MAINTAINERS b/vendor/github.com/kelseyhightower/envconfig/MAINTAINERS new file mode 100644 index 0000000000000..6527a9f2cc274 --- /dev/null +++ b/vendor/github.com/kelseyhightower/envconfig/MAINTAINERS @@ -0,0 +1,2 @@ +Kelsey Hightower kelsey.hightower@gmail.com github.com/kelseyhightower +Travis Parker travis.parker@gmail.com github.com/teepark diff --git a/vendor/github.com/kelseyhightower/envconfig/README.md b/vendor/github.com/kelseyhightower/envconfig/README.md new file mode 100644 index 0000000000000..33408d645e4bd --- /dev/null +++ b/vendor/github.com/kelseyhightower/envconfig/README.md @@ -0,0 +1,192 @@ +# envconfig + +[![Build Status](https://travis-ci.org/kelseyhightower/envconfig.svg)](https://travis-ci.org/kelseyhightower/envconfig) + +```Go +import "github.com/kelseyhightower/envconfig" +``` + +## Documentation + +See [godoc](http://godoc.org/github.com/kelseyhightower/envconfig) + +## Usage + +Set some environment variables: + +```Bash +export MYAPP_DEBUG=false +export MYAPP_PORT=8080 +export MYAPP_USER=Kelsey +export MYAPP_RATE="0.5" +export MYAPP_TIMEOUT="3m" +export MYAPP_USERS="rob,ken,robert" +export MYAPP_COLORCODES="red:1,green:2,blue:3" +``` + +Write some code: + +```Go +package main + +import ( + "fmt" + "log" + "time" + + "github.com/kelseyhightower/envconfig" +) + +type Specification struct { + Debug bool + Port int + User string + Users []string + Rate float32 + Timeout time.Duration + ColorCodes map[string]int +} + +func main() { + var s Specification + err := envconfig.Process("myapp", &s) + if err != nil { + log.Fatal(err.Error()) + } + format := "Debug: %v\nPort: %d\nUser: %s\nRate: %f\nTimeout: %s\n" + _, err = fmt.Printf(format, s.Debug, s.Port, s.User, s.Rate, s.Timeout) + if err != nil { + log.Fatal(err.Error()) + } + + fmt.Println("Users:") + for _, u := range s.Users { + fmt.Printf(" %s\n", u) + } + + fmt.Println("Color codes:") + for k, v := range s.ColorCodes { + fmt.Printf(" %s: %d\n", k, v) + } +} +``` + +Results: + +```Bash +Debug: false +Port: 8080 +User: Kelsey +Rate: 0.500000 +Timeout: 3m0s +Users: + rob + ken + robert +Color codes: + red: 1 + green: 2 + blue: 3 +``` + +## Struct Tag Support + +Envconfig supports the use of struct tags to specify alternate, default, and required +environment variables. + +For example, consider the following struct: + +```Go +type Specification struct { + ManualOverride1 string `envconfig:"manual_override_1"` + DefaultVar string `default:"foobar"` + RequiredVar string `required:"true"` + IgnoredVar string `ignored:"true"` + AutoSplitVar string `split_words:"true"` + RequiredAndAutoSplitVar string `required:"true" split_words:"true"` +} +``` + +Envconfig has automatic support for CamelCased struct elements when the +`split_words:"true"` tag is supplied. Without this tag, `AutoSplitVar` above +would look for an environment variable called `MYAPP_AUTOSPLITVAR`. With the +setting applied it will look for `MYAPP_AUTO_SPLIT_VAR`. Note that numbers +will get globbed into the previous word. If the setting does not do the +right thing, you may use a manual override. + +Envconfig will process value for `ManualOverride1` by populating it with the +value for `MYAPP_MANUAL_OVERRIDE_1`. Without this struct tag, it would have +instead looked up `MYAPP_MANUALOVERRIDE1`. With the `split_words:"true"` tag +it would have looked up `MYAPP_MANUAL_OVERRIDE1`. + +```Bash +export MYAPP_MANUAL_OVERRIDE_1="this will be the value" + +# export MYAPP_MANUALOVERRIDE1="and this will not" +``` + +If envconfig can't find an environment variable value for `MYAPP_DEFAULTVAR`, +it will populate it with "foobar" as a default value. + +If envconfig can't find an environment variable value for `MYAPP_REQUIREDVAR`, +it will return an error when asked to process the struct. If +`MYAPP_REQUIREDVAR` is present but empty, envconfig will not return an error. + +If envconfig can't find an environment variable in the form `PREFIX_MYVAR`, and there +is a struct tag defined, it will try to populate your variable with an environment +variable that directly matches the envconfig tag in your struct definition: + +```shell +export SERVICE_HOST=127.0.0.1 +export MYAPP_DEBUG=true +``` +```Go +type Specification struct { + ServiceHost string `envconfig:"SERVICE_HOST"` + Debug bool +} +``` + +Envconfig won't process a field with the "ignored" tag set to "true", even if a corresponding +environment variable is set. + +## Supported Struct Field Types + +envconfig supports these struct field types: + + * string + * int8, int16, int32, int64 + * bool + * float32, float64 + * slices of any supported type + * maps (keys and values of any supported type) + * [encoding.TextUnmarshaler](https://golang.org/pkg/encoding/#TextUnmarshaler) + * [encoding.BinaryUnmarshaler](https://golang.org/pkg/encoding/#BinaryUnmarshaler) + * [time.Duration](https://golang.org/pkg/time/#Duration) + +Embedded structs using these fields are also supported. + +## Custom Decoders + +Any field whose type (or pointer-to-type) implements `envconfig.Decoder` can +control its own deserialization: + +```Bash +export DNS_SERVER=8.8.8.8 +``` + +```Go +type IPDecoder net.IP + +func (ipd *IPDecoder) Decode(value string) error { + *ipd = IPDecoder(net.ParseIP(value)) + return nil +} + +type DNSConfig struct { + Address IPDecoder `envconfig:"DNS_SERVER"` +} +``` + +Also, envconfig will use a `Set(string) error` method like from the +[flag.Value](https://godoc.org/flag#Value) interface if implemented. diff --git a/vendor/github.com/kelseyhightower/envconfig/doc.go b/vendor/github.com/kelseyhightower/envconfig/doc.go new file mode 100644 index 0000000000000..f28561cd1cbc4 --- /dev/null +++ b/vendor/github.com/kelseyhightower/envconfig/doc.go @@ -0,0 +1,8 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the MIT License that can be found in +// the LICENSE file. + +// Package envconfig implements decoding of environment variables based on a user +// defined specification. A typical use is using environment variables for +// configuration settings. +package envconfig diff --git a/vendor/github.com/kelseyhightower/envconfig/env_os.go b/vendor/github.com/kelseyhightower/envconfig/env_os.go new file mode 100644 index 0000000000000..eba07a6c6130e --- /dev/null +++ b/vendor/github.com/kelseyhightower/envconfig/env_os.go @@ -0,0 +1,7 @@ +// +build appengine go1.5 + +package envconfig + +import "os" + +var lookupEnv = os.LookupEnv diff --git a/vendor/github.com/kelseyhightower/envconfig/env_syscall.go b/vendor/github.com/kelseyhightower/envconfig/env_syscall.go new file mode 100644 index 0000000000000..4254540080777 --- /dev/null +++ b/vendor/github.com/kelseyhightower/envconfig/env_syscall.go @@ -0,0 +1,7 @@ +// +build !appengine,!go1.5 + +package envconfig + +import "syscall" + +var lookupEnv = syscall.Getenv diff --git a/vendor/github.com/kelseyhightower/envconfig/envconfig.go b/vendor/github.com/kelseyhightower/envconfig/envconfig.go new file mode 100644 index 0000000000000..3f16108db8a46 --- /dev/null +++ b/vendor/github.com/kelseyhightower/envconfig/envconfig.go @@ -0,0 +1,382 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the MIT License that can be found in +// the LICENSE file. + +package envconfig + +import ( + "encoding" + "errors" + "fmt" + "os" + "reflect" + "regexp" + "strconv" + "strings" + "time" +) + +// ErrInvalidSpecification indicates that a specification is of the wrong type. +var ErrInvalidSpecification = errors.New("specification must be a struct pointer") + +var gatherRegexp = regexp.MustCompile("([^A-Z]+|[A-Z]+[^A-Z]+|[A-Z]+)") +var acronymRegexp = regexp.MustCompile("([A-Z]+)([A-Z][^A-Z]+)") + +// A ParseError occurs when an environment variable cannot be converted to +// the type required by a struct field during assignment. +type ParseError struct { + KeyName string + FieldName string + TypeName string + Value string + Err error +} + +// Decoder has the same semantics as Setter, but takes higher precedence. +// It is provided for historical compatibility. +type Decoder interface { + Decode(value string) error +} + +// Setter is implemented by types can self-deserialize values. +// Any type that implements flag.Value also implements Setter. +type Setter interface { + Set(value string) error +} + +func (e *ParseError) Error() string { + return fmt.Sprintf("envconfig.Process: assigning %[1]s to %[2]s: converting '%[3]s' to type %[4]s. details: %[5]s", e.KeyName, e.FieldName, e.Value, e.TypeName, e.Err) +} + +// varInfo maintains information about the configuration variable +type varInfo struct { + Name string + Alt string + Key string + Field reflect.Value + Tags reflect.StructTag +} + +// GatherInfo gathers information about the specified struct +func gatherInfo(prefix string, spec interface{}) ([]varInfo, error) { + s := reflect.ValueOf(spec) + + if s.Kind() != reflect.Ptr { + return nil, ErrInvalidSpecification + } + s = s.Elem() + if s.Kind() != reflect.Struct { + return nil, ErrInvalidSpecification + } + typeOfSpec := s.Type() + + // over allocate an info array, we will extend if needed later + infos := make([]varInfo, 0, s.NumField()) + for i := 0; i < s.NumField(); i++ { + f := s.Field(i) + ftype := typeOfSpec.Field(i) + if !f.CanSet() || isTrue(ftype.Tag.Get("ignored")) { + continue + } + + for f.Kind() == reflect.Ptr { + if f.IsNil() { + if f.Type().Elem().Kind() != reflect.Struct { + // nil pointer to a non-struct: leave it alone + break + } + // nil pointer to struct: create a zero instance + f.Set(reflect.New(f.Type().Elem())) + } + f = f.Elem() + } + + // Capture information about the config variable + info := varInfo{ + Name: ftype.Name, + Field: f, + Tags: ftype.Tag, + Alt: strings.ToUpper(ftype.Tag.Get("envconfig")), + } + + // Default to the field name as the env var name (will be upcased) + info.Key = info.Name + + // Best effort to un-pick camel casing as separate words + if isTrue(ftype.Tag.Get("split_words")) { + words := gatherRegexp.FindAllStringSubmatch(ftype.Name, -1) + if len(words) > 0 { + var name []string + for _, words := range words { + if m := acronymRegexp.FindStringSubmatch(words[0]); len(m) == 3 { + name = append(name, m[1], m[2]) + } else { + name = append(name, words[0]) + } + } + + info.Key = strings.Join(name, "_") + } + } + if info.Alt != "" { + info.Key = info.Alt + } + if prefix != "" { + info.Key = fmt.Sprintf("%s_%s", prefix, info.Key) + } + info.Key = strings.ToUpper(info.Key) + infos = append(infos, info) + + if f.Kind() == reflect.Struct { + // honor Decode if present + if decoderFrom(f) == nil && setterFrom(f) == nil && textUnmarshaler(f) == nil && binaryUnmarshaler(f) == nil { + innerPrefix := prefix + if !ftype.Anonymous { + innerPrefix = info.Key + } + + embeddedPtr := f.Addr().Interface() + embeddedInfos, err := gatherInfo(innerPrefix, embeddedPtr) + if err != nil { + return nil, err + } + infos = append(infos[:len(infos)-1], embeddedInfos...) + + continue + } + } + } + return infos, nil +} + +// CheckDisallowed checks that no environment variables with the prefix are set +// that we don't know how or want to parse. This is likely only meaningful with +// a non-empty prefix. +func CheckDisallowed(prefix string, spec interface{}) error { + infos, err := gatherInfo(prefix, spec) + if err != nil { + return err + } + + vars := make(map[string]struct{}) + for _, info := range infos { + vars[info.Key] = struct{}{} + } + + if prefix != "" { + prefix = strings.ToUpper(prefix) + "_" + } + + for _, env := range os.Environ() { + if !strings.HasPrefix(env, prefix) { + continue + } + v := strings.SplitN(env, "=", 2)[0] + if _, found := vars[v]; !found { + return fmt.Errorf("unknown environment variable %s", v) + } + } + + return nil +} + +// Process populates the specified struct based on environment variables +func Process(prefix string, spec interface{}) error { + infos, err := gatherInfo(prefix, spec) + + for _, info := range infos { + + // `os.Getenv` cannot differentiate between an explicitly set empty value + // and an unset value. `os.LookupEnv` is preferred to `syscall.Getenv`, + // but it is only available in go1.5 or newer. We're using Go build tags + // here to use os.LookupEnv for >=go1.5 + value, ok := lookupEnv(info.Key) + if !ok && info.Alt != "" { + value, ok = lookupEnv(info.Alt) + } + + def := info.Tags.Get("default") + if def != "" && !ok { + value = def + } + + req := info.Tags.Get("required") + if !ok && def == "" { + if isTrue(req) { + key := info.Key + if info.Alt != "" { + key = info.Alt + } + return fmt.Errorf("required key %s missing value", key) + } + continue + } + + err = processField(value, info.Field) + if err != nil { + return &ParseError{ + KeyName: info.Key, + FieldName: info.Name, + TypeName: info.Field.Type().String(), + Value: value, + Err: err, + } + } + } + + return err +} + +// MustProcess is the same as Process but panics if an error occurs +func MustProcess(prefix string, spec interface{}) { + if err := Process(prefix, spec); err != nil { + panic(err) + } +} + +func processField(value string, field reflect.Value) error { + typ := field.Type() + + decoder := decoderFrom(field) + if decoder != nil { + return decoder.Decode(value) + } + // look for Set method if Decode not defined + setter := setterFrom(field) + if setter != nil { + return setter.Set(value) + } + + if t := textUnmarshaler(field); t != nil { + return t.UnmarshalText([]byte(value)) + } + + if b := binaryUnmarshaler(field); b != nil { + return b.UnmarshalBinary([]byte(value)) + } + + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + if field.IsNil() { + field.Set(reflect.New(typ)) + } + field = field.Elem() + } + + switch typ.Kind() { + case reflect.String: + field.SetString(value) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + var ( + val int64 + err error + ) + if field.Kind() == reflect.Int64 && typ.PkgPath() == "time" && typ.Name() == "Duration" { + var d time.Duration + d, err = time.ParseDuration(value) + val = int64(d) + } else { + val, err = strconv.ParseInt(value, 0, typ.Bits()) + } + if err != nil { + return err + } + + field.SetInt(val) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + val, err := strconv.ParseUint(value, 0, typ.Bits()) + if err != nil { + return err + } + field.SetUint(val) + case reflect.Bool: + val, err := strconv.ParseBool(value) + if err != nil { + return err + } + field.SetBool(val) + case reflect.Float32, reflect.Float64: + val, err := strconv.ParseFloat(value, typ.Bits()) + if err != nil { + return err + } + field.SetFloat(val) + case reflect.Slice: + sl := reflect.MakeSlice(typ, 0, 0) + if typ.Elem().Kind() == reflect.Uint8 { + sl = reflect.ValueOf([]byte(value)) + } else if len(strings.TrimSpace(value)) != 0 { + vals := strings.Split(value, ",") + sl = reflect.MakeSlice(typ, len(vals), len(vals)) + for i, val := range vals { + err := processField(val, sl.Index(i)) + if err != nil { + return err + } + } + } + field.Set(sl) + case reflect.Map: + mp := reflect.MakeMap(typ) + if len(strings.TrimSpace(value)) != 0 { + pairs := strings.Split(value, ",") + for _, pair := range pairs { + kvpair := strings.Split(pair, ":") + if len(kvpair) != 2 { + return fmt.Errorf("invalid map item: %q", pair) + } + k := reflect.New(typ.Key()).Elem() + err := processField(kvpair[0], k) + if err != nil { + return err + } + v := reflect.New(typ.Elem()).Elem() + err = processField(kvpair[1], v) + if err != nil { + return err + } + mp.SetMapIndex(k, v) + } + } + field.Set(mp) + } + + return nil +} + +func interfaceFrom(field reflect.Value, fn func(interface{}, *bool)) { + // it may be impossible for a struct field to fail this check + if !field.CanInterface() { + return + } + var ok bool + fn(field.Interface(), &ok) + if !ok && field.CanAddr() { + fn(field.Addr().Interface(), &ok) + } +} + +func decoderFrom(field reflect.Value) (d Decoder) { + interfaceFrom(field, func(v interface{}, ok *bool) { d, *ok = v.(Decoder) }) + return d +} + +func setterFrom(field reflect.Value) (s Setter) { + interfaceFrom(field, func(v interface{}, ok *bool) { s, *ok = v.(Setter) }) + return s +} + +func textUnmarshaler(field reflect.Value) (t encoding.TextUnmarshaler) { + interfaceFrom(field, func(v interface{}, ok *bool) { t, *ok = v.(encoding.TextUnmarshaler) }) + return t +} + +func binaryUnmarshaler(field reflect.Value) (b encoding.BinaryUnmarshaler) { + interfaceFrom(field, func(v interface{}, ok *bool) { b, *ok = v.(encoding.BinaryUnmarshaler) }) + return b +} + +func isTrue(s string) bool { + b, _ := strconv.ParseBool(s) + return b +} diff --git a/vendor/github.com/kelseyhightower/envconfig/go.mod b/vendor/github.com/kelseyhightower/envconfig/go.mod new file mode 100644 index 0000000000000..1561d1e4e048c --- /dev/null +++ b/vendor/github.com/kelseyhightower/envconfig/go.mod @@ -0,0 +1 @@ +module github.com/kelseyhightower/envconfig diff --git a/vendor/github.com/kelseyhightower/envconfig/usage.go b/vendor/github.com/kelseyhightower/envconfig/usage.go new file mode 100644 index 0000000000000..1e6d0a8f367c4 --- /dev/null +++ b/vendor/github.com/kelseyhightower/envconfig/usage.go @@ -0,0 +1,164 @@ +// Copyright (c) 2016 Kelsey Hightower and others. All rights reserved. +// Use of this source code is governed by the MIT License that can be found in +// the LICENSE file. + +package envconfig + +import ( + "encoding" + "fmt" + "io" + "os" + "reflect" + "strconv" + "strings" + "text/tabwriter" + "text/template" +) + +const ( + // DefaultListFormat constant to use to display usage in a list format + DefaultListFormat = `This application is configured via the environment. The following environment +variables can be used: +{{range .}} +{{usage_key .}} + [description] {{usage_description .}} + [type] {{usage_type .}} + [default] {{usage_default .}} + [required] {{usage_required .}}{{end}} +` + // DefaultTableFormat constant to use to display usage in a tabular format + DefaultTableFormat = `This application is configured via the environment. The following environment +variables can be used: + +KEY TYPE DEFAULT REQUIRED DESCRIPTION +{{range .}}{{usage_key .}} {{usage_type .}} {{usage_default .}} {{usage_required .}} {{usage_description .}} +{{end}}` +) + +var ( + decoderType = reflect.TypeOf((*Decoder)(nil)).Elem() + setterType = reflect.TypeOf((*Setter)(nil)).Elem() + textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() + binaryUnmarshalerType = reflect.TypeOf((*encoding.BinaryUnmarshaler)(nil)).Elem() +) + +func implementsInterface(t reflect.Type) bool { + return t.Implements(decoderType) || + reflect.PtrTo(t).Implements(decoderType) || + t.Implements(setterType) || + reflect.PtrTo(t).Implements(setterType) || + t.Implements(textUnmarshalerType) || + reflect.PtrTo(t).Implements(textUnmarshalerType) || + t.Implements(binaryUnmarshalerType) || + reflect.PtrTo(t).Implements(binaryUnmarshalerType) +} + +// toTypeDescription converts Go types into a human readable description +func toTypeDescription(t reflect.Type) string { + switch t.Kind() { + case reflect.Array, reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return "String" + } + return fmt.Sprintf("Comma-separated list of %s", toTypeDescription(t.Elem())) + case reflect.Map: + return fmt.Sprintf( + "Comma-separated list of %s:%s pairs", + toTypeDescription(t.Key()), + toTypeDescription(t.Elem()), + ) + case reflect.Ptr: + return toTypeDescription(t.Elem()) + case reflect.Struct: + if implementsInterface(t) && t.Name() != "" { + return t.Name() + } + return "" + case reflect.String: + name := t.Name() + if name != "" && name != "string" { + return name + } + return "String" + case reflect.Bool: + name := t.Name() + if name != "" && name != "bool" { + return name + } + return "True or False" + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + name := t.Name() + if name != "" && !strings.HasPrefix(name, "int") { + return name + } + return "Integer" + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + name := t.Name() + if name != "" && !strings.HasPrefix(name, "uint") { + return name + } + return "Unsigned Integer" + case reflect.Float32, reflect.Float64: + name := t.Name() + if name != "" && !strings.HasPrefix(name, "float") { + return name + } + return "Float" + } + return fmt.Sprintf("%+v", t) +} + +// Usage writes usage information to stdout using the default header and table format +func Usage(prefix string, spec interface{}) error { + // The default is to output the usage information as a table + // Create tabwriter instance to support table output + tabs := tabwriter.NewWriter(os.Stdout, 1, 0, 4, ' ', 0) + + err := Usagef(prefix, spec, tabs, DefaultTableFormat) + tabs.Flush() + return err +} + +// Usagef writes usage information to the specified io.Writer using the specifed template specification +func Usagef(prefix string, spec interface{}, out io.Writer, format string) error { + + // Specify the default usage template functions + functions := template.FuncMap{ + "usage_key": func(v varInfo) string { return v.Key }, + "usage_description": func(v varInfo) string { return v.Tags.Get("desc") }, + "usage_type": func(v varInfo) string { return toTypeDescription(v.Field.Type()) }, + "usage_default": func(v varInfo) string { return v.Tags.Get("default") }, + "usage_required": func(v varInfo) (string, error) { + req := v.Tags.Get("required") + if req != "" { + reqB, err := strconv.ParseBool(req) + if err != nil { + return "", err + } + if reqB { + req = "true" + } + } + return req, nil + }, + } + + tmpl, err := template.New("envconfig").Funcs(functions).Parse(format) + if err != nil { + return err + } + + return Usaget(prefix, spec, out, tmpl) +} + +// Usaget writes usage information to the specified io.Writer using the specified template +func Usaget(prefix string, spec interface{}, out io.Writer, tmpl *template.Template) error { + // gather first + infos, err := gatherInfo(prefix, spec) + if err != nil { + return err + } + + return tmpl.Execute(out, infos) +} From 79cd766819fa07cb4838b0b8239b58e14ca111b6 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Tue, 10 Sep 2019 21:46:58 +0200 Subject: [PATCH 02/18] feat: pkg/cfg Adds configuration package for managing various configuration sources with correct precedence --- pkg/cfg/cfg.go | 26 ++++++++++++++ pkg/cfg/env.go | 12 +++++++ pkg/cfg/files.go | 55 ++++++++++++++++++++++++++++ pkg/cfg/flag.go | 94 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 187 insertions(+) create mode 100644 pkg/cfg/cfg.go create mode 100644 pkg/cfg/env.go create mode 100644 pkg/cfg/files.go create mode 100644 pkg/cfg/flag.go diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go new file mode 100644 index 0000000000000..319a0e5659c3e --- /dev/null +++ b/pkg/cfg/cfg.go @@ -0,0 +1,26 @@ +package cfg + +import ( + "github.com/pkg/errors" +) + +// Source is a generic configuration source. This function may do whatever is +// required to obtain the configuration. It is passed a pointer to the +// destination, which will be something compatible to `json.Unmarshal`. The +// obtained configuration may be written to this object, it may also contain +// data from previous sources. +type Source func(interface{}) error + +// Unmarshal merges the values of the various configuration sources and sets them on +// `dst`. The object must be compatible with `json.Unmarshal`. +func Unmarshal(dst interface{}, sources ...Source) error { + if len(sources) == 0 { + panic("No sources supplied to cfg.Unmarshal(). This is most likely a programming issue and should never happen. Check the code!") + } + for _, source := range sources { + if err := source(dst); err != nil { + return errors.Wrap(err, "sourcing") + } + } + return nil +} diff --git a/pkg/cfg/env.go b/pkg/cfg/env.go new file mode 100644 index 0000000000000..ff6064f66859d --- /dev/null +++ b/pkg/cfg/env.go @@ -0,0 +1,12 @@ +package cfg + +import ( + "github.com/kelseyhightower/envconfig" +) + +// Env returns a Source that takes configuration from environment variables. +func Env(prefix string) Source { + return func(dst interface{}) error { + return envconfig.Process(prefix, dst) + } +} diff --git a/pkg/cfg/files.go b/pkg/cfg/files.go new file mode 100644 index 0000000000000..d5a1a85d6464a --- /dev/null +++ b/pkg/cfg/files.go @@ -0,0 +1,55 @@ +package cfg + +import ( + "encoding/json" + "flag" + "io/ioutil" + + yaml "gopkg.in/yaml.v2" +) + +// JSON returns a Source that opens the supplied `.json` file and loads it. +func JSON(f *string) Source { + return func(dst interface{}) error { + + j, err := ioutil.ReadFile(*f) + if err != nil { + return err + } + + if err := json.Unmarshal(j, dst); err != nil { + return err + } + return nil + } +} + +// YAML returns a Source that opens the supplied `.yaml` file and loads it. +func YAML(f *string) Source { + return func(dst interface{}) error { + if f == nil { + return nil + } + + y, err := ioutil.ReadFile(*f) + if err != nil { + return err + } + + if err := yaml.Unmarshal(y, dst); err != nil { + return err + } + return nil + } +} + +// YAMLFlag defines a `config.file` flag and loads this file +func YAMLFlag() Source { + f := flag.String("config.file", "", ".yaml configuration file to parse") + return func(dst interface{}) error { + if *f == "" { + f = nil + } + return YAML(f)(dst) + } +} diff --git a/pkg/cfg/flag.go b/pkg/cfg/flag.go new file mode 100644 index 0000000000000..1f48dcef1def1 --- /dev/null +++ b/pkg/cfg/flag.go @@ -0,0 +1,94 @@ +package cfg + +import ( + "errors" + "flag" + "reflect" + + "github.com/cortexproject/cortex/pkg/util/flagext" + ghodss "github.com/ghodss/yaml" + yaml "gopkg.in/yaml.v2" +) + +// FlagDefaultsDangerous obtains defaults from a flag Registerer and DANGEROUSLY +// replaces (!!) the dst object with reg. +// Make sure this is always the first source, or it will overwrite other data. +// The defaults are set into ptr, if is an empty byte slice and not nil. +func FlagDefaultsDangerous(reg flagext.Registerer, ptr *[]byte) Source { + // some cortex code uses global `flag.*Var`. To avoid the flag package panicking, the default flagset is temporarily swapped out + tmp := flag.CommandLine + flag.CommandLine = flag.NewFlagSet("", flag.ContinueOnError) + flagext.DefaultValues(reg) + flag.CommandLine = tmp + + d, err := yaml.Marshal(reg) + if err != nil { + panic(err) + } + + if ptr != nil { + *ptr = d + } + + return func(dst interface{}) error { + v := reflect.ValueOf(dst) + if v.Kind() != reflect.Ptr { + return errors.New("dst not a pointer") + } + v = reflect.Indirect(v) + v.Set(reflect.Indirect(reflect.ValueOf(reg))) + return nil + } +} + +// Flags returns a source that sets values from changed (!) flags onto dst. +// The must have the same structure as dst, usually it is even the same type. +// The defaults slice MUST contain valid defaults as yaml from the same registerer. +func Flags(reg flagext.Registerer, defaults []byte) Source { + flagext.RegisterFlags(reg) + flag.Parse() + data, err := yaml.Marshal(reg) + if err != nil { + panic(err) + } + return func(dst interface{}) error { + return stripDefaults(dst, data, defaults) + } +} + +// stripDefaults takes the final config, the config from flags and the defaults, +// but merges only non-default values onto dst +func stripDefaults(dst interface{}, flagsStr, defStr []byte) error { + // using ghodss.Unmarshal to receive map[string]interface{} + var live, def map[string]interface{} + if err := ghodss.Unmarshal(flagsStr, &live); err != nil { + return err + } + if err := ghodss.Unmarshal(defStr, &def); err != nil { + return err + } + + res := deleteSame(live, def, "") + + out, err := yaml.Marshal(res) + if err != nil { + return err + } + return yaml.Unmarshal(out, dst) +} + +// deleteSame filters all equal values out of deeply nested map[string]interface{}'s +func deleteSame(live, def map[string]interface{}, path string) map[string]interface{} { + for k := range live { + if reflect.DeepEqual(live[k], def[k]) { + delete(live, k) + continue + } + + if _, ok := live[k].(map[string]interface{}); ok { + live[k] = deleteSame(live[k].(map[string]interface{}), def[k].(map[string]interface{}), path+"."+k) + } + + } + return live +} From de9555f07b71c826bc9f7357bcc4ce578ecc4159 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Tue, 10 Sep 2019 21:47:41 +0200 Subject: [PATCH 03/18] feat(loki): pkg/cfg --- cmd/loki/main.go | 58 +++++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/cmd/loki/main.go b/cmd/loki/main.go index c061280360d50..c37f51d79372b 100644 --- a/cmd/loki/main.go +++ b/cmd/loki/main.go @@ -3,11 +3,12 @@ package main import ( "flag" "fmt" + "log" "os" "reflect" "github.com/go-kit/kit/log/level" - "github.com/grafana/loki/pkg/helpers" + "github.com/grafana/loki/pkg/cfg" "github.com/grafana/loki/pkg/loki" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/version" @@ -15,9 +16,6 @@ import ( "github.com/weaveworks/common/tracing" "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/flagext" - - "github.com/grafana/loki/pkg/util/validation" ) func init() { @@ -25,40 +23,14 @@ func init() { } func main() { - var ( - cfg loki.Config - configFile = "" - ) - flag.StringVar(&configFile, "config.file", "", "Configuration file to load.") - flagext.RegisterFlags(&cfg) - printVersion := flag.Bool("version", false, "Print this builds version information") - flag.Parse() - + cfg := loadConfig() if *printVersion { fmt.Print(version.Print("loki")) os.Exit(0) } - // LimitsConfig has a customer UnmarshalYAML that will set the defaults to a global. - // This global is set to the config passed into the last call to `NewOverrides`. If we don't - // call it atleast once, the defaults are set to an empty struct. - // We call it with the flag values so that the config file unmarshalling only overrides the values set in the config. - if _, err := validation.NewOverrides(cfg.LimitsConfig); err != nil { - level.Error(util.Logger).Log("msg", "error loading limits", "err", err) - os.Exit(1) - } - - util.InitLogger(&cfg.Server) - - if configFile != "" { - if err := helpers.LoadConfig(configFile, &cfg); err != nil { - level.Error(util.Logger).Log("msg", "error loading config", "filename", configFile, "err", err) - os.Exit(1) - } - } - - // Re-init the logger which will now honor a different log level set in cfg.Server + // Init the logger which will honor the log level set in cfg.Server if reflect.DeepEqual(&cfg.Server.LogLevel, &logging.Level{}) { level.Error(util.Logger).Log("msg", "invalid log level") os.Exit(1) @@ -74,7 +46,8 @@ func main() { } }() - t, err := loki.New(cfg) + // Start Loki + t, err := loki.New(*cfg) if err != nil { level.Error(util.Logger).Log("msg", "error initialising loki", "err", err) os.Exit(1) @@ -91,3 +64,22 @@ func main() { os.Exit(1) } } + +// loadConfig loads the config from various sources which take precedence over each other +func loadConfig() *loki.Config { + var config loki.Config + + // defaults shared by FlagDefaultsDangerous and Flags + var defaults []byte + + // unmarshal config + if err := cfg.Unmarshal(&config, + cfg.FlagDefaultsDangerous(&loki.Config{}, &defaults), + cfg.YAMLFlag(), + cfg.Flags(&loki.Config{}, defaults), + ); err != nil { + log.Fatalln(err) + } + + return &config +} From 11167e547e79871b5bcb2cc75b9f56f94b340900 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Tue, 10 Sep 2019 21:47:53 +0200 Subject: [PATCH 04/18] feat(promtail): pkg/cfg --- cmd/promtail/main.go | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/cmd/promtail/main.go b/cmd/promtail/main.go index 4933ebc8244b3..3bf8f1a6714a3 100644 --- a/cmd/promtail/main.go +++ b/cmd/promtail/main.go @@ -3,17 +3,17 @@ package main import ( "flag" "fmt" + "log" "os" "reflect" "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/version" "github.com/weaveworks/common/logging" - "github.com/grafana/loki/pkg/helpers" + "github.com/grafana/loki/pkg/cfg" "github.com/grafana/loki/pkg/logentry/stages" "github.com/grafana/loki/pkg/promtail" "github.com/grafana/loki/pkg/promtail/config" @@ -24,31 +24,15 @@ func init() { } func main() { - var ( - configFile = "cmd/promtail/promtail-local-config.yaml" - config config.Config - ) - flag.StringVar(&configFile, "config.file", "promtail.yml", "The config file.") - flagext.RegisterFlags(&config) - printVersion := flag.Bool("version", false, "Print this builds version information") - flag.Parse() + config := loadConfig() if *printVersion { fmt.Print(version.Print("promtail")) os.Exit(0) } - util.InitLogger(&config.ServerConfig.Config) - - if configFile != "" { - if err := helpers.LoadConfig(configFile, &config); err != nil { - level.Error(util.Logger).Log("msg", "error loading config", "filename", configFile, "err", err) - os.Exit(1) - } - } - - // Re-init the logger which will now honor a different log level set in ServerConfig.Config + // Init the logger which will honor the log level set in cfg.Server if reflect.DeepEqual(&config.ServerConfig.Config.LogLevel, &logging.Level{}) { level.Error(util.Logger).Log("msg", "invalid log level") os.Exit(1) @@ -61,7 +45,7 @@ func main() { stages.Debug = true } - p, err := promtail.New(config) + p, err := promtail.New(*config) if err != nil { level.Error(util.Logger).Log("msg", "error creating promtail", "error", err) os.Exit(1) @@ -76,3 +60,18 @@ func main() { p.Shutdown() } + +func loadConfig() *config.Config { + var c config.Config + var defaults []byte + + if err := cfg.Unmarshal(&c, + cfg.FlagDefaultsDangerous(&config.Config{}, &defaults), + cfg.YAMLFlag(), + cfg.Flags(&config.Config{}, defaults), + ); err != nil { + log.Fatalln(err) + } + + return &c +} From c4c404885993afd61c82e618de96b8b9987d7453 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 12 Sep 2019 19:40:04 +0200 Subject: [PATCH 05/18] fix(cfg): remove env support Not immediately needed and easy to reimplement, removed for now --- Gopkg.lock | 9 - pkg/cfg/env.go | 12 - .../kelseyhightower/envconfig/.travis.yml | 13 - .../kelseyhightower/envconfig/LICENSE | 19 - .../kelseyhightower/envconfig/MAINTAINERS | 2 - .../kelseyhightower/envconfig/README.md | 192 --------- .../kelseyhightower/envconfig/doc.go | 8 - .../kelseyhightower/envconfig/env_os.go | 7 - .../kelseyhightower/envconfig/env_syscall.go | 7 - .../kelseyhightower/envconfig/envconfig.go | 382 ------------------ .../kelseyhightower/envconfig/go.mod | 1 - .../kelseyhightower/envconfig/usage.go | 164 -------- 12 files changed, 816 deletions(-) delete mode 100644 pkg/cfg/env.go delete mode 100644 vendor/github.com/kelseyhightower/envconfig/.travis.yml delete mode 100644 vendor/github.com/kelseyhightower/envconfig/LICENSE delete mode 100644 vendor/github.com/kelseyhightower/envconfig/MAINTAINERS delete mode 100644 vendor/github.com/kelseyhightower/envconfig/README.md delete mode 100644 vendor/github.com/kelseyhightower/envconfig/doc.go delete mode 100644 vendor/github.com/kelseyhightower/envconfig/env_os.go delete mode 100644 vendor/github.com/kelseyhightower/envconfig/env_syscall.go delete mode 100644 vendor/github.com/kelseyhightower/envconfig/envconfig.go delete mode 100644 vendor/github.com/kelseyhightower/envconfig/go.mod delete mode 100644 vendor/github.com/kelseyhightower/envconfig/usage.go diff --git a/Gopkg.lock b/Gopkg.lock index 98587ca01d2db..09fc44d8d13c5 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -741,14 +741,6 @@ revision = "1624edc4454b8682399def8740d46db5e4362ba4" version = "v1.1.5" -[[projects]] - digest = "1:fd9bea48bbc5bba66d9891c72af7255fbebecdff845c37c679406174ece5ca1b" - name = "github.com/kelseyhightower/envconfig" - packages = ["."] - pruneopts = "UT" - revision = "0b417c4ec4a8a82eecc22a1459a504aa55163d61" - version = "v1.4.0" - [[projects]] digest = "1:9b67e6f39db9eb120a6c5977e4b29f33d7e01970777e77152ed79395f7559291" name = "github.com/klauspost/compress" @@ -1870,7 +1862,6 @@ "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc", "github.com/hpcloud/tail", "github.com/jmespath/go-jmespath", - "github.com/kelseyhightower/envconfig", "github.com/klauspost/compress/gzip", "github.com/mitchellh/mapstructure", "github.com/opentracing/opentracing-go", diff --git a/pkg/cfg/env.go b/pkg/cfg/env.go deleted file mode 100644 index ff6064f66859d..0000000000000 --- a/pkg/cfg/env.go +++ /dev/null @@ -1,12 +0,0 @@ -package cfg - -import ( - "github.com/kelseyhightower/envconfig" -) - -// Env returns a Source that takes configuration from environment variables. -func Env(prefix string) Source { - return func(dst interface{}) error { - return envconfig.Process(prefix, dst) - } -} diff --git a/vendor/github.com/kelseyhightower/envconfig/.travis.yml b/vendor/github.com/kelseyhightower/envconfig/.travis.yml deleted file mode 100644 index 04b97aed6164e..0000000000000 --- a/vendor/github.com/kelseyhightower/envconfig/.travis.yml +++ /dev/null @@ -1,13 +0,0 @@ -language: go - -go: - - 1.4.x - - 1.5.x - - 1.6.x - - 1.7.x - - 1.8.x - - 1.9.x - - 1.10.x - - 1.11.x - - 1.12.x - - tip diff --git a/vendor/github.com/kelseyhightower/envconfig/LICENSE b/vendor/github.com/kelseyhightower/envconfig/LICENSE deleted file mode 100644 index 4bfa7a84d818d..0000000000000 --- a/vendor/github.com/kelseyhightower/envconfig/LICENSE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2013 Kelsey Hightower - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/kelseyhightower/envconfig/MAINTAINERS b/vendor/github.com/kelseyhightower/envconfig/MAINTAINERS deleted file mode 100644 index 6527a9f2cc274..0000000000000 --- a/vendor/github.com/kelseyhightower/envconfig/MAINTAINERS +++ /dev/null @@ -1,2 +0,0 @@ -Kelsey Hightower kelsey.hightower@gmail.com github.com/kelseyhightower -Travis Parker travis.parker@gmail.com github.com/teepark diff --git a/vendor/github.com/kelseyhightower/envconfig/README.md b/vendor/github.com/kelseyhightower/envconfig/README.md deleted file mode 100644 index 33408d645e4bd..0000000000000 --- a/vendor/github.com/kelseyhightower/envconfig/README.md +++ /dev/null @@ -1,192 +0,0 @@ -# envconfig - -[![Build Status](https://travis-ci.org/kelseyhightower/envconfig.svg)](https://travis-ci.org/kelseyhightower/envconfig) - -```Go -import "github.com/kelseyhightower/envconfig" -``` - -## Documentation - -See [godoc](http://godoc.org/github.com/kelseyhightower/envconfig) - -## Usage - -Set some environment variables: - -```Bash -export MYAPP_DEBUG=false -export MYAPP_PORT=8080 -export MYAPP_USER=Kelsey -export MYAPP_RATE="0.5" -export MYAPP_TIMEOUT="3m" -export MYAPP_USERS="rob,ken,robert" -export MYAPP_COLORCODES="red:1,green:2,blue:3" -``` - -Write some code: - -```Go -package main - -import ( - "fmt" - "log" - "time" - - "github.com/kelseyhightower/envconfig" -) - -type Specification struct { - Debug bool - Port int - User string - Users []string - Rate float32 - Timeout time.Duration - ColorCodes map[string]int -} - -func main() { - var s Specification - err := envconfig.Process("myapp", &s) - if err != nil { - log.Fatal(err.Error()) - } - format := "Debug: %v\nPort: %d\nUser: %s\nRate: %f\nTimeout: %s\n" - _, err = fmt.Printf(format, s.Debug, s.Port, s.User, s.Rate, s.Timeout) - if err != nil { - log.Fatal(err.Error()) - } - - fmt.Println("Users:") - for _, u := range s.Users { - fmt.Printf(" %s\n", u) - } - - fmt.Println("Color codes:") - for k, v := range s.ColorCodes { - fmt.Printf(" %s: %d\n", k, v) - } -} -``` - -Results: - -```Bash -Debug: false -Port: 8080 -User: Kelsey -Rate: 0.500000 -Timeout: 3m0s -Users: - rob - ken - robert -Color codes: - red: 1 - green: 2 - blue: 3 -``` - -## Struct Tag Support - -Envconfig supports the use of struct tags to specify alternate, default, and required -environment variables. - -For example, consider the following struct: - -```Go -type Specification struct { - ManualOverride1 string `envconfig:"manual_override_1"` - DefaultVar string `default:"foobar"` - RequiredVar string `required:"true"` - IgnoredVar string `ignored:"true"` - AutoSplitVar string `split_words:"true"` - RequiredAndAutoSplitVar string `required:"true" split_words:"true"` -} -``` - -Envconfig has automatic support for CamelCased struct elements when the -`split_words:"true"` tag is supplied. Without this tag, `AutoSplitVar` above -would look for an environment variable called `MYAPP_AUTOSPLITVAR`. With the -setting applied it will look for `MYAPP_AUTO_SPLIT_VAR`. Note that numbers -will get globbed into the previous word. If the setting does not do the -right thing, you may use a manual override. - -Envconfig will process value for `ManualOverride1` by populating it with the -value for `MYAPP_MANUAL_OVERRIDE_1`. Without this struct tag, it would have -instead looked up `MYAPP_MANUALOVERRIDE1`. With the `split_words:"true"` tag -it would have looked up `MYAPP_MANUAL_OVERRIDE1`. - -```Bash -export MYAPP_MANUAL_OVERRIDE_1="this will be the value" - -# export MYAPP_MANUALOVERRIDE1="and this will not" -``` - -If envconfig can't find an environment variable value for `MYAPP_DEFAULTVAR`, -it will populate it with "foobar" as a default value. - -If envconfig can't find an environment variable value for `MYAPP_REQUIREDVAR`, -it will return an error when asked to process the struct. If -`MYAPP_REQUIREDVAR` is present but empty, envconfig will not return an error. - -If envconfig can't find an environment variable in the form `PREFIX_MYVAR`, and there -is a struct tag defined, it will try to populate your variable with an environment -variable that directly matches the envconfig tag in your struct definition: - -```shell -export SERVICE_HOST=127.0.0.1 -export MYAPP_DEBUG=true -``` -```Go -type Specification struct { - ServiceHost string `envconfig:"SERVICE_HOST"` - Debug bool -} -``` - -Envconfig won't process a field with the "ignored" tag set to "true", even if a corresponding -environment variable is set. - -## Supported Struct Field Types - -envconfig supports these struct field types: - - * string - * int8, int16, int32, int64 - * bool - * float32, float64 - * slices of any supported type - * maps (keys and values of any supported type) - * [encoding.TextUnmarshaler](https://golang.org/pkg/encoding/#TextUnmarshaler) - * [encoding.BinaryUnmarshaler](https://golang.org/pkg/encoding/#BinaryUnmarshaler) - * [time.Duration](https://golang.org/pkg/time/#Duration) - -Embedded structs using these fields are also supported. - -## Custom Decoders - -Any field whose type (or pointer-to-type) implements `envconfig.Decoder` can -control its own deserialization: - -```Bash -export DNS_SERVER=8.8.8.8 -``` - -```Go -type IPDecoder net.IP - -func (ipd *IPDecoder) Decode(value string) error { - *ipd = IPDecoder(net.ParseIP(value)) - return nil -} - -type DNSConfig struct { - Address IPDecoder `envconfig:"DNS_SERVER"` -} -``` - -Also, envconfig will use a `Set(string) error` method like from the -[flag.Value](https://godoc.org/flag#Value) interface if implemented. diff --git a/vendor/github.com/kelseyhightower/envconfig/doc.go b/vendor/github.com/kelseyhightower/envconfig/doc.go deleted file mode 100644 index f28561cd1cbc4..0000000000000 --- a/vendor/github.com/kelseyhightower/envconfig/doc.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2013 Kelsey Hightower. All rights reserved. -// Use of this source code is governed by the MIT License that can be found in -// the LICENSE file. - -// Package envconfig implements decoding of environment variables based on a user -// defined specification. A typical use is using environment variables for -// configuration settings. -package envconfig diff --git a/vendor/github.com/kelseyhightower/envconfig/env_os.go b/vendor/github.com/kelseyhightower/envconfig/env_os.go deleted file mode 100644 index eba07a6c6130e..0000000000000 --- a/vendor/github.com/kelseyhightower/envconfig/env_os.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build appengine go1.5 - -package envconfig - -import "os" - -var lookupEnv = os.LookupEnv diff --git a/vendor/github.com/kelseyhightower/envconfig/env_syscall.go b/vendor/github.com/kelseyhightower/envconfig/env_syscall.go deleted file mode 100644 index 4254540080777..0000000000000 --- a/vendor/github.com/kelseyhightower/envconfig/env_syscall.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build !appengine,!go1.5 - -package envconfig - -import "syscall" - -var lookupEnv = syscall.Getenv diff --git a/vendor/github.com/kelseyhightower/envconfig/envconfig.go b/vendor/github.com/kelseyhightower/envconfig/envconfig.go deleted file mode 100644 index 3f16108db8a46..0000000000000 --- a/vendor/github.com/kelseyhightower/envconfig/envconfig.go +++ /dev/null @@ -1,382 +0,0 @@ -// Copyright (c) 2013 Kelsey Hightower. All rights reserved. -// Use of this source code is governed by the MIT License that can be found in -// the LICENSE file. - -package envconfig - -import ( - "encoding" - "errors" - "fmt" - "os" - "reflect" - "regexp" - "strconv" - "strings" - "time" -) - -// ErrInvalidSpecification indicates that a specification is of the wrong type. -var ErrInvalidSpecification = errors.New("specification must be a struct pointer") - -var gatherRegexp = regexp.MustCompile("([^A-Z]+|[A-Z]+[^A-Z]+|[A-Z]+)") -var acronymRegexp = regexp.MustCompile("([A-Z]+)([A-Z][^A-Z]+)") - -// A ParseError occurs when an environment variable cannot be converted to -// the type required by a struct field during assignment. -type ParseError struct { - KeyName string - FieldName string - TypeName string - Value string - Err error -} - -// Decoder has the same semantics as Setter, but takes higher precedence. -// It is provided for historical compatibility. -type Decoder interface { - Decode(value string) error -} - -// Setter is implemented by types can self-deserialize values. -// Any type that implements flag.Value also implements Setter. -type Setter interface { - Set(value string) error -} - -func (e *ParseError) Error() string { - return fmt.Sprintf("envconfig.Process: assigning %[1]s to %[2]s: converting '%[3]s' to type %[4]s. details: %[5]s", e.KeyName, e.FieldName, e.Value, e.TypeName, e.Err) -} - -// varInfo maintains information about the configuration variable -type varInfo struct { - Name string - Alt string - Key string - Field reflect.Value - Tags reflect.StructTag -} - -// GatherInfo gathers information about the specified struct -func gatherInfo(prefix string, spec interface{}) ([]varInfo, error) { - s := reflect.ValueOf(spec) - - if s.Kind() != reflect.Ptr { - return nil, ErrInvalidSpecification - } - s = s.Elem() - if s.Kind() != reflect.Struct { - return nil, ErrInvalidSpecification - } - typeOfSpec := s.Type() - - // over allocate an info array, we will extend if needed later - infos := make([]varInfo, 0, s.NumField()) - for i := 0; i < s.NumField(); i++ { - f := s.Field(i) - ftype := typeOfSpec.Field(i) - if !f.CanSet() || isTrue(ftype.Tag.Get("ignored")) { - continue - } - - for f.Kind() == reflect.Ptr { - if f.IsNil() { - if f.Type().Elem().Kind() != reflect.Struct { - // nil pointer to a non-struct: leave it alone - break - } - // nil pointer to struct: create a zero instance - f.Set(reflect.New(f.Type().Elem())) - } - f = f.Elem() - } - - // Capture information about the config variable - info := varInfo{ - Name: ftype.Name, - Field: f, - Tags: ftype.Tag, - Alt: strings.ToUpper(ftype.Tag.Get("envconfig")), - } - - // Default to the field name as the env var name (will be upcased) - info.Key = info.Name - - // Best effort to un-pick camel casing as separate words - if isTrue(ftype.Tag.Get("split_words")) { - words := gatherRegexp.FindAllStringSubmatch(ftype.Name, -1) - if len(words) > 0 { - var name []string - for _, words := range words { - if m := acronymRegexp.FindStringSubmatch(words[0]); len(m) == 3 { - name = append(name, m[1], m[2]) - } else { - name = append(name, words[0]) - } - } - - info.Key = strings.Join(name, "_") - } - } - if info.Alt != "" { - info.Key = info.Alt - } - if prefix != "" { - info.Key = fmt.Sprintf("%s_%s", prefix, info.Key) - } - info.Key = strings.ToUpper(info.Key) - infos = append(infos, info) - - if f.Kind() == reflect.Struct { - // honor Decode if present - if decoderFrom(f) == nil && setterFrom(f) == nil && textUnmarshaler(f) == nil && binaryUnmarshaler(f) == nil { - innerPrefix := prefix - if !ftype.Anonymous { - innerPrefix = info.Key - } - - embeddedPtr := f.Addr().Interface() - embeddedInfos, err := gatherInfo(innerPrefix, embeddedPtr) - if err != nil { - return nil, err - } - infos = append(infos[:len(infos)-1], embeddedInfos...) - - continue - } - } - } - return infos, nil -} - -// CheckDisallowed checks that no environment variables with the prefix are set -// that we don't know how or want to parse. This is likely only meaningful with -// a non-empty prefix. -func CheckDisallowed(prefix string, spec interface{}) error { - infos, err := gatherInfo(prefix, spec) - if err != nil { - return err - } - - vars := make(map[string]struct{}) - for _, info := range infos { - vars[info.Key] = struct{}{} - } - - if prefix != "" { - prefix = strings.ToUpper(prefix) + "_" - } - - for _, env := range os.Environ() { - if !strings.HasPrefix(env, prefix) { - continue - } - v := strings.SplitN(env, "=", 2)[0] - if _, found := vars[v]; !found { - return fmt.Errorf("unknown environment variable %s", v) - } - } - - return nil -} - -// Process populates the specified struct based on environment variables -func Process(prefix string, spec interface{}) error { - infos, err := gatherInfo(prefix, spec) - - for _, info := range infos { - - // `os.Getenv` cannot differentiate between an explicitly set empty value - // and an unset value. `os.LookupEnv` is preferred to `syscall.Getenv`, - // but it is only available in go1.5 or newer. We're using Go build tags - // here to use os.LookupEnv for >=go1.5 - value, ok := lookupEnv(info.Key) - if !ok && info.Alt != "" { - value, ok = lookupEnv(info.Alt) - } - - def := info.Tags.Get("default") - if def != "" && !ok { - value = def - } - - req := info.Tags.Get("required") - if !ok && def == "" { - if isTrue(req) { - key := info.Key - if info.Alt != "" { - key = info.Alt - } - return fmt.Errorf("required key %s missing value", key) - } - continue - } - - err = processField(value, info.Field) - if err != nil { - return &ParseError{ - KeyName: info.Key, - FieldName: info.Name, - TypeName: info.Field.Type().String(), - Value: value, - Err: err, - } - } - } - - return err -} - -// MustProcess is the same as Process but panics if an error occurs -func MustProcess(prefix string, spec interface{}) { - if err := Process(prefix, spec); err != nil { - panic(err) - } -} - -func processField(value string, field reflect.Value) error { - typ := field.Type() - - decoder := decoderFrom(field) - if decoder != nil { - return decoder.Decode(value) - } - // look for Set method if Decode not defined - setter := setterFrom(field) - if setter != nil { - return setter.Set(value) - } - - if t := textUnmarshaler(field); t != nil { - return t.UnmarshalText([]byte(value)) - } - - if b := binaryUnmarshaler(field); b != nil { - return b.UnmarshalBinary([]byte(value)) - } - - if typ.Kind() == reflect.Ptr { - typ = typ.Elem() - if field.IsNil() { - field.Set(reflect.New(typ)) - } - field = field.Elem() - } - - switch typ.Kind() { - case reflect.String: - field.SetString(value) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - var ( - val int64 - err error - ) - if field.Kind() == reflect.Int64 && typ.PkgPath() == "time" && typ.Name() == "Duration" { - var d time.Duration - d, err = time.ParseDuration(value) - val = int64(d) - } else { - val, err = strconv.ParseInt(value, 0, typ.Bits()) - } - if err != nil { - return err - } - - field.SetInt(val) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - val, err := strconv.ParseUint(value, 0, typ.Bits()) - if err != nil { - return err - } - field.SetUint(val) - case reflect.Bool: - val, err := strconv.ParseBool(value) - if err != nil { - return err - } - field.SetBool(val) - case reflect.Float32, reflect.Float64: - val, err := strconv.ParseFloat(value, typ.Bits()) - if err != nil { - return err - } - field.SetFloat(val) - case reflect.Slice: - sl := reflect.MakeSlice(typ, 0, 0) - if typ.Elem().Kind() == reflect.Uint8 { - sl = reflect.ValueOf([]byte(value)) - } else if len(strings.TrimSpace(value)) != 0 { - vals := strings.Split(value, ",") - sl = reflect.MakeSlice(typ, len(vals), len(vals)) - for i, val := range vals { - err := processField(val, sl.Index(i)) - if err != nil { - return err - } - } - } - field.Set(sl) - case reflect.Map: - mp := reflect.MakeMap(typ) - if len(strings.TrimSpace(value)) != 0 { - pairs := strings.Split(value, ",") - for _, pair := range pairs { - kvpair := strings.Split(pair, ":") - if len(kvpair) != 2 { - return fmt.Errorf("invalid map item: %q", pair) - } - k := reflect.New(typ.Key()).Elem() - err := processField(kvpair[0], k) - if err != nil { - return err - } - v := reflect.New(typ.Elem()).Elem() - err = processField(kvpair[1], v) - if err != nil { - return err - } - mp.SetMapIndex(k, v) - } - } - field.Set(mp) - } - - return nil -} - -func interfaceFrom(field reflect.Value, fn func(interface{}, *bool)) { - // it may be impossible for a struct field to fail this check - if !field.CanInterface() { - return - } - var ok bool - fn(field.Interface(), &ok) - if !ok && field.CanAddr() { - fn(field.Addr().Interface(), &ok) - } -} - -func decoderFrom(field reflect.Value) (d Decoder) { - interfaceFrom(field, func(v interface{}, ok *bool) { d, *ok = v.(Decoder) }) - return d -} - -func setterFrom(field reflect.Value) (s Setter) { - interfaceFrom(field, func(v interface{}, ok *bool) { s, *ok = v.(Setter) }) - return s -} - -func textUnmarshaler(field reflect.Value) (t encoding.TextUnmarshaler) { - interfaceFrom(field, func(v interface{}, ok *bool) { t, *ok = v.(encoding.TextUnmarshaler) }) - return t -} - -func binaryUnmarshaler(field reflect.Value) (b encoding.BinaryUnmarshaler) { - interfaceFrom(field, func(v interface{}, ok *bool) { b, *ok = v.(encoding.BinaryUnmarshaler) }) - return b -} - -func isTrue(s string) bool { - b, _ := strconv.ParseBool(s) - return b -} diff --git a/vendor/github.com/kelseyhightower/envconfig/go.mod b/vendor/github.com/kelseyhightower/envconfig/go.mod deleted file mode 100644 index 1561d1e4e048c..0000000000000 --- a/vendor/github.com/kelseyhightower/envconfig/go.mod +++ /dev/null @@ -1 +0,0 @@ -module github.com/kelseyhightower/envconfig diff --git a/vendor/github.com/kelseyhightower/envconfig/usage.go b/vendor/github.com/kelseyhightower/envconfig/usage.go deleted file mode 100644 index 1e6d0a8f367c4..0000000000000 --- a/vendor/github.com/kelseyhightower/envconfig/usage.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (c) 2016 Kelsey Hightower and others. All rights reserved. -// Use of this source code is governed by the MIT License that can be found in -// the LICENSE file. - -package envconfig - -import ( - "encoding" - "fmt" - "io" - "os" - "reflect" - "strconv" - "strings" - "text/tabwriter" - "text/template" -) - -const ( - // DefaultListFormat constant to use to display usage in a list format - DefaultListFormat = `This application is configured via the environment. The following environment -variables can be used: -{{range .}} -{{usage_key .}} - [description] {{usage_description .}} - [type] {{usage_type .}} - [default] {{usage_default .}} - [required] {{usage_required .}}{{end}} -` - // DefaultTableFormat constant to use to display usage in a tabular format - DefaultTableFormat = `This application is configured via the environment. The following environment -variables can be used: - -KEY TYPE DEFAULT REQUIRED DESCRIPTION -{{range .}}{{usage_key .}} {{usage_type .}} {{usage_default .}} {{usage_required .}} {{usage_description .}} -{{end}}` -) - -var ( - decoderType = reflect.TypeOf((*Decoder)(nil)).Elem() - setterType = reflect.TypeOf((*Setter)(nil)).Elem() - textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() - binaryUnmarshalerType = reflect.TypeOf((*encoding.BinaryUnmarshaler)(nil)).Elem() -) - -func implementsInterface(t reflect.Type) bool { - return t.Implements(decoderType) || - reflect.PtrTo(t).Implements(decoderType) || - t.Implements(setterType) || - reflect.PtrTo(t).Implements(setterType) || - t.Implements(textUnmarshalerType) || - reflect.PtrTo(t).Implements(textUnmarshalerType) || - t.Implements(binaryUnmarshalerType) || - reflect.PtrTo(t).Implements(binaryUnmarshalerType) -} - -// toTypeDescription converts Go types into a human readable description -func toTypeDescription(t reflect.Type) string { - switch t.Kind() { - case reflect.Array, reflect.Slice: - if t.Elem().Kind() == reflect.Uint8 { - return "String" - } - return fmt.Sprintf("Comma-separated list of %s", toTypeDescription(t.Elem())) - case reflect.Map: - return fmt.Sprintf( - "Comma-separated list of %s:%s pairs", - toTypeDescription(t.Key()), - toTypeDescription(t.Elem()), - ) - case reflect.Ptr: - return toTypeDescription(t.Elem()) - case reflect.Struct: - if implementsInterface(t) && t.Name() != "" { - return t.Name() - } - return "" - case reflect.String: - name := t.Name() - if name != "" && name != "string" { - return name - } - return "String" - case reflect.Bool: - name := t.Name() - if name != "" && name != "bool" { - return name - } - return "True or False" - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - name := t.Name() - if name != "" && !strings.HasPrefix(name, "int") { - return name - } - return "Integer" - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - name := t.Name() - if name != "" && !strings.HasPrefix(name, "uint") { - return name - } - return "Unsigned Integer" - case reflect.Float32, reflect.Float64: - name := t.Name() - if name != "" && !strings.HasPrefix(name, "float") { - return name - } - return "Float" - } - return fmt.Sprintf("%+v", t) -} - -// Usage writes usage information to stdout using the default header and table format -func Usage(prefix string, spec interface{}) error { - // The default is to output the usage information as a table - // Create tabwriter instance to support table output - tabs := tabwriter.NewWriter(os.Stdout, 1, 0, 4, ' ', 0) - - err := Usagef(prefix, spec, tabs, DefaultTableFormat) - tabs.Flush() - return err -} - -// Usagef writes usage information to the specified io.Writer using the specifed template specification -func Usagef(prefix string, spec interface{}, out io.Writer, format string) error { - - // Specify the default usage template functions - functions := template.FuncMap{ - "usage_key": func(v varInfo) string { return v.Key }, - "usage_description": func(v varInfo) string { return v.Tags.Get("desc") }, - "usage_type": func(v varInfo) string { return toTypeDescription(v.Field.Type()) }, - "usage_default": func(v varInfo) string { return v.Tags.Get("default") }, - "usage_required": func(v varInfo) (string, error) { - req := v.Tags.Get("required") - if req != "" { - reqB, err := strconv.ParseBool(req) - if err != nil { - return "", err - } - if reqB { - req = "true" - } - } - return req, nil - }, - } - - tmpl, err := template.New("envconfig").Funcs(functions).Parse(format) - if err != nil { - return err - } - - return Usaget(prefix, spec, out, tmpl) -} - -// Usaget writes usage information to the specified io.Writer using the specified template -func Usaget(prefix string, spec interface{}, out io.Writer, tmpl *template.Template) error { - // gather first - infos, err := gatherInfo(prefix, spec) - if err != nil { - return err - } - - return tmpl.Execute(out, infos) -} From e8e538ba6b110ca21e91ab9032df77adefc71ddb Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 12 Sep 2019 21:44:34 +0200 Subject: [PATCH 06/18] feat(cfg): Parse() wrapper Adds a convenience wrapper `cfg.Parse()` which unmarshals from flag and yaml --- cmd/loki/main.go | 35 ++++++++++------------------------- cmd/promtail/main.go | 22 +++++----------------- pkg/cfg/cfg.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/cmd/loki/main.go b/cmd/loki/main.go index c37f51d79372b..750abb0e572a6 100644 --- a/cmd/loki/main.go +++ b/cmd/loki/main.go @@ -24,21 +24,25 @@ func init() { func main() { printVersion := flag.Bool("version", false, "Print this builds version information") - cfg := loadConfig() + + var config loki.Config + if err := cfg.Parse(&config); err != nil { + log.Fatalln(err) + } if *printVersion { fmt.Print(version.Print("loki")) os.Exit(0) } - // Init the logger which will honor the log level set in cfg.Server - if reflect.DeepEqual(&cfg.Server.LogLevel, &logging.Level{}) { + // Init the logger which will honor the log level set in config.Server + if reflect.DeepEqual(&config.Server.LogLevel, &logging.Level{}) { level.Error(util.Logger).Log("msg", "invalid log level") os.Exit(1) } - util.InitLogger(&cfg.Server) + util.InitLogger(&config.Server) // Setting the environment variable JAEGER_AGENT_HOST enables tracing - trace := tracing.NewFromEnv(fmt.Sprintf("loki-%s", cfg.Target)) + trace := tracing.NewFromEnv(fmt.Sprintf("loki-%s", config.Target)) defer func() { if err := trace.Close(); err != nil { level.Error(util.Logger).Log("msg", "error closing tracing", "err", err) @@ -47,7 +51,7 @@ func main() { }() // Start Loki - t, err := loki.New(*cfg) + t, err := loki.New(config) if err != nil { level.Error(util.Logger).Log("msg", "error initialising loki", "err", err) os.Exit(1) @@ -64,22 +68,3 @@ func main() { os.Exit(1) } } - -// loadConfig loads the config from various sources which take precedence over each other -func loadConfig() *loki.Config { - var config loki.Config - - // defaults shared by FlagDefaultsDangerous and Flags - var defaults []byte - - // unmarshal config - if err := cfg.Unmarshal(&config, - cfg.FlagDefaultsDangerous(&loki.Config{}, &defaults), - cfg.YAMLFlag(), - cfg.Flags(&loki.Config{}, defaults), - ); err != nil { - log.Fatalln(err) - } - - return &config -} diff --git a/cmd/promtail/main.go b/cmd/promtail/main.go index 3bf8f1a6714a3..51ef9b4b2cf6d 100644 --- a/cmd/promtail/main.go +++ b/cmd/promtail/main.go @@ -25,8 +25,11 @@ func init() { func main() { printVersion := flag.Bool("version", false, "Print this builds version information") - config := loadConfig() + var config config.Config + if err := cfg.Parse(&config); err != nil { + log.Fatalln(err) + } if *printVersion { fmt.Print(version.Print("promtail")) os.Exit(0) @@ -45,7 +48,7 @@ func main() { stages.Debug = true } - p, err := promtail.New(*config) + p, err := promtail.New(config) if err != nil { level.Error(util.Logger).Log("msg", "error creating promtail", "error", err) os.Exit(1) @@ -60,18 +63,3 @@ func main() { p.Shutdown() } - -func loadConfig() *config.Config { - var c config.Config - var defaults []byte - - if err := cfg.Unmarshal(&c, - cfg.FlagDefaultsDangerous(&config.Config{}, &defaults), - cfg.YAMLFlag(), - cfg.Flags(&config.Config{}, defaults), - ); err != nil { - log.Fatalln(err) - } - - return &c -} diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go index 319a0e5659c3e..d37bc115ac301 100644 --- a/pkg/cfg/cfg.go +++ b/pkg/cfg/cfg.go @@ -1,6 +1,9 @@ package cfg import ( + "reflect" + + "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/pkg/errors" ) @@ -24,3 +27,29 @@ func Unmarshal(dst interface{}, sources ...Source) error { } return nil } + +// Parse is a higher level wrapper for Unmarshal that automatically parses flags and a .yaml file +func Parse(dst interface{}) error { + // check dst is a pointer + v := reflect.ValueOf(dst) + if v.Kind() != reflect.Ptr { + panic("dst not a pointer") + } + + // obtain type of dst for cloning + t := reflect.Indirect(v).Type() + + // create new instances of dst's type for flags + d := reflect.New(t).Interface().(flagext.Registerer) + f := reflect.New(t).Interface().(flagext.Registerer) + + // shared state for FlagDefaultsDangerous and Flags + var defaultsYaml []byte + + // unmarshal config + return Unmarshal(dst, + FlagDefaultsDangerous(d, &defaultsYaml), + YAMLFlag(), + Flags(f, defaultsYaml), + ) +} From d1640ea3a99d6d658f0e7ae99849180011aa0f43 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 12 Sep 2019 22:17:38 +0200 Subject: [PATCH 07/18] test(cfg): flag unit tests --- pkg/cfg/cfg.go | 10 +++++- pkg/cfg/data_test.go | 32 +++++++++++++++++ pkg/cfg/files.go | 24 ++++++++----- pkg/cfg/flag.go | 28 ++++++++------- pkg/cfg/flag_test.go | 83 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+), 21 deletions(-) create mode 100644 pkg/cfg/data_test.go create mode 100644 pkg/cfg/flag_test.go diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go index d37bc115ac301..e92dc5b70201a 100644 --- a/pkg/cfg/cfg.go +++ b/pkg/cfg/cfg.go @@ -14,12 +14,20 @@ import ( // data from previous sources. type Source func(interface{}) error +var ( + ErrNotPointer = errors.New("dst is not a pointer") +) + // Unmarshal merges the values of the various configuration sources and sets them on // `dst`. The object must be compatible with `json.Unmarshal`. func Unmarshal(dst interface{}, sources ...Source) error { if len(sources) == 0 { panic("No sources supplied to cfg.Unmarshal(). This is most likely a programming issue and should never happen. Check the code!") } + if reflect.ValueOf(dst).Kind() != reflect.Ptr { + return ErrNotPointer + } + for _, source := range sources { if err := source(dst); err != nil { return errors.Wrap(err, "sourcing") @@ -33,7 +41,7 @@ func Parse(dst interface{}) error { // check dst is a pointer v := reflect.ValueOf(dst) if v.Kind() != reflect.Ptr { - panic("dst not a pointer") + return ErrNotPointer } // obtain type of dst for cloning diff --git a/pkg/cfg/data_test.go b/pkg/cfg/data_test.go new file mode 100644 index 0000000000000..080886cf81ac6 --- /dev/null +++ b/pkg/cfg/data_test.go @@ -0,0 +1,32 @@ +package cfg + +import ( + "flag" + "time" +) + +// Data is a test Data structure +type Data struct { + Verbose bool `yaml:"verbose"` + Server Server `yaml:"server"` + TLS TLS `yaml:"tls"` +} + +type Server struct { + Port int `yaml:"port"` + Timeout time.Duration `yaml:"timeout"` +} + +type TLS struct { + Cert string `yaml:"cert"` + Key string `yaml:"key"` +} + +func (d *Data) RegisterFlags(fs *flag.FlagSet) { + fs.BoolVar(&d.Verbose, "verbose", false, "") + fs.IntVar(&d.Server.Port, "server.port", 80, "") + fs.DurationVar(&d.Server.Timeout, "server.timeout", 60*time.Second, "") + + fs.StringVar(&d.TLS.Cert, "tls.cert", "CERT", "") + fs.StringVar(&d.TLS.Key, "tls.key", "KEY", "") +} diff --git a/pkg/cfg/files.go b/pkg/cfg/files.go index d5a1a85d6464a..a2cbcc0c92c76 100644 --- a/pkg/cfg/files.go +++ b/pkg/cfg/files.go @@ -17,10 +17,14 @@ func JSON(f *string) Source { return err } - if err := json.Unmarshal(j, dst); err != nil { - return err - } - return nil + return dJSON(j)(dst) + } +} + +// dJSON returns a JSON source and allows dependency injection +func dJSON(y []byte) Source { + return func(dst interface{}) error { + return json.Unmarshal(y, dst) } } @@ -36,10 +40,14 @@ func YAML(f *string) Source { return err } - if err := yaml.Unmarshal(y, dst); err != nil { - return err - } - return nil + return dYAML(y)(dst) + } +} + +// dYAML returns a YAML source and allows dependency injection +func dYAML(y []byte) Source { + return func(dst interface{}) error { + return yaml.Unmarshal(y, dst) } } diff --git a/pkg/cfg/flag.go b/pkg/cfg/flag.go index 1f48dcef1def1..5b7d30a9d25ba 100644 --- a/pkg/cfg/flag.go +++ b/pkg/cfg/flag.go @@ -1,8 +1,8 @@ package cfg import ( - "errors" "flag" + "os" "reflect" "github.com/cortexproject/cortex/pkg/util/flagext" @@ -21,21 +21,19 @@ func FlagDefaultsDangerous(reg flagext.Registerer, ptr *[]byte) Source { flagext.DefaultValues(reg) flag.CommandLine = tmp - d, err := yaml.Marshal(reg) - if err != nil { - panic(err) - } - + // dump defaults for subsequent `Flags()` call if ptr != nil { + d, err := yaml.Marshal(reg) + if err != nil { + panic(err) + } + *ptr = d } return func(dst interface{}) error { - v := reflect.ValueOf(dst) - if v.Kind() != reflect.Ptr { - return errors.New("dst not a pointer") - } - v = reflect.Indirect(v) + // set *reg for *dst + v := reflect.Indirect(reflect.ValueOf(dst)) v.Set(reflect.Indirect(reflect.ValueOf(reg))) return nil } @@ -45,8 +43,14 @@ func FlagDefaultsDangerous(reg flagext.Registerer, ptr *[]byte) Source { // The must have the same structure as dst, usually it is even the same type. // The defaults slice MUST contain valid defaults as yaml from the same registerer. func Flags(reg flagext.Registerer, defaults []byte) Source { + return dFlags(os.Args[1:], reg, defaults) +} + +// dFlags parses flags defined by reg using the args slice onto dst. +// The defaults slice MUST contain valid defaults as yaml from the same registerer. +func dFlags(args []string, reg flagext.Registerer, defaults []byte) Source { flagext.RegisterFlags(reg) - flag.Parse() + _ = flag.CommandLine.Parse(args) data, err := yaml.Marshal(reg) if err != nil { panic(err) diff --git a/pkg/cfg/flag_test.go b/pkg/cfg/flag_test.go new file mode 100644 index 0000000000000..eac116fd5cce4 --- /dev/null +++ b/pkg/cfg/flag_test.go @@ -0,0 +1,83 @@ +package cfg + +import ( + "flag" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// NOTE: These tests CANNOT run in parallel, because of the global state in the +// `flag` package of the standard library. + +// TestDefaults checks whether `FlagDefaultsDangerous()` correctly sets values from flag defaults +func TestDefaults(t *testing.T) { + var d Data + err := FlagDefaultsDangerous(&Data{}, nil)(&d) + require.NoError(t, err) + assert.Equal(t, Data{ + Verbose: false, + Server: Server{ + Port: 80, + Timeout: 60 * time.Second, + }, + TLS: TLS{ + Cert: "CERT", + Key: "KEY", + }, + }, d) +} + +// TestFlagsSetOnly checks that user-supplied flag values can be correctly distinguished from defaults +func TestFlagsSetOnly(t *testing.T) { + flag.CommandLine = &flag.FlagSet{} + var c Data + + var sharedMem []byte + // c is not passed, to only see the output of dFlags() afterwards + err := FlagDefaultsDangerous(&Data{}, &sharedMem)(&Data{}) + require.NoError(t, err) + + err = dFlags([]string{"-verbose", "-server.timeout=12h"}, &Data{}, sharedMem)(&c) + require.NoError(t, err) + + // check that defaults are correctly stripped away + assert.Equal(t, Data{ + Verbose: true, + Server: Server{ + Port: 0, + Timeout: 12 * time.Hour, + }, + TLS: TLS{ + Cert: "", + Key: "", + }, + }, c) +} + +// TestFlagsMerge checks that defaults and user-supplied values merge correctly +func TestFlagsMerge(t *testing.T) { + flag.CommandLine = &flag.FlagSet{} + + var c Data + var sharedMem []byte + + err := Unmarshal(&c, + FlagDefaultsDangerous(&Data{}, &sharedMem), + dFlags([]string{"-verbose", "-server.timeout=12h"}, &Data{}, sharedMem), + ) + require.NoError(t, err) + assert.Equal(t, Data{ + Verbose: true, + Server: Server{ + Port: 80, + Timeout: 12 * time.Hour, + }, + TLS: TLS{ + Cert: "CERT", + Key: "KEY", + }, + }, c) +} From 503938dc48af9e85675c2f7354e23e8489a9395d Mon Sep 17 00:00:00 2001 From: sh0rez Date: Sat, 14 Sep 2019 15:02:56 +0200 Subject: [PATCH 08/18] test(cfg): Parse(), FlagDefaults() Adds tests for Parse(), furthermore introduces FlagDefaults, a safer wrapper for FlagDefaultsDangerous that fails when dst is not zero valued. --- pkg/cfg/cfg.go | 20 +++++++++++++++++--- pkg/cfg/cfg_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ pkg/cfg/files.go | 4 ++-- pkg/cfg/flag.go | 17 +++++++++++++++++ pkg/cfg/flag_test.go | 8 ++++---- 5 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 pkg/cfg/cfg_test.go diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go index e92dc5b70201a..89a74ceb54373 100644 --- a/pkg/cfg/cfg.go +++ b/pkg/cfg/cfg.go @@ -38,6 +38,20 @@ func Unmarshal(dst interface{}, sources ...Source) error { // Parse is a higher level wrapper for Unmarshal that automatically parses flags and a .yaml file func Parse(dst interface{}) error { + yamlSource := func() Source { + return YAMLFlag("config.file", "", ".yaml configuration file to parse") + } + flagSource := func(reg flagext.Registerer, def []byte) Source { + return Flags(reg, def) + } + return dParse(dst, yamlSource, flagSource) +} + +// dParse is like Parse, but allows dependency injection +func dParse(dst interface{}, + yamlSource func() Source, + flagSource func(flagext.Registerer, []byte) Source, +) error { // check dst is a pointer v := reflect.ValueOf(dst) if v.Kind() != reflect.Ptr { @@ -56,8 +70,8 @@ func Parse(dst interface{}) error { // unmarshal config return Unmarshal(dst, - FlagDefaultsDangerous(d, &defaultsYaml), - YAMLFlag(), - Flags(f, defaultsYaml), + FlagDefaults(d, &defaultsYaml), + yamlSource(), + flagSource(f, defaultsYaml), ) } diff --git a/pkg/cfg/cfg_test.go b/pkg/cfg/cfg_test.go new file mode 100644 index 0000000000000..1626c0733c507 --- /dev/null +++ b/pkg/cfg/cfg_test.go @@ -0,0 +1,41 @@ +package cfg + +import ( + "testing" + "time" + + "github.com/cortexproject/cortex/pkg/util/flagext" + "github.com/stretchr/testify/require" +) + +func TestParse(t *testing.T) { + yamlSource := func() Source { + return dYAML([]byte(` +server: + port: 2000 + timeout: 60h +tls: + key: YAML +`)) + } + + flagSource := func(reg flagext.Registerer, def []byte) Source { + return dFlags([]string{"-verbose", "-server.port=21"}, reg, def) + } + + var c Data + err := dParse(&c, yamlSource, flagSource) + require.NoError(t, err) + + require.Equal(t, Data{ + Verbose: true, + Server: Server{ + Port: 21, + Timeout: 60 * time.Hour, + }, + TLS: TLS{ + Cert: "CERT", + Key: "YAML", + }, + }, c) +} diff --git a/pkg/cfg/files.go b/pkg/cfg/files.go index a2cbcc0c92c76..ee74114d7c564 100644 --- a/pkg/cfg/files.go +++ b/pkg/cfg/files.go @@ -52,8 +52,8 @@ func dYAML(y []byte) Source { } // YAMLFlag defines a `config.file` flag and loads this file -func YAMLFlag() Source { - f := flag.String("config.file", "", ".yaml configuration file to parse") +func YAMLFlag(name, value, help string) Source { + f := flag.String(name, value, help) return func(dst interface{}) error { if *f == "" { f = nil diff --git a/pkg/cfg/flag.go b/pkg/cfg/flag.go index 5b7d30a9d25ba..636370d52488c 100644 --- a/pkg/cfg/flag.go +++ b/pkg/cfg/flag.go @@ -7,9 +7,26 @@ import ( "github.com/cortexproject/cortex/pkg/util/flagext" ghodss "github.com/ghodss/yaml" + "github.com/pkg/errors" yaml "gopkg.in/yaml.v2" ) +// FlagDefaults obtains defaults from a flag registerer and fails when it is not the first source. +func FlagDefaults(reg flagext.Registerer, ptr *[]byte) Source { + f := FlagDefaultsDangerous(reg, ptr) + + return func(dst interface{}) error { + // check dst is it's zero value (also indirect pointers first) + v := reflect.Indirect(reflect.ValueOf(dst)) + z := reflect.Zero(v.Type()) + if !reflect.DeepEqual(v.Interface(), z.Interface()) { + return errors.New("FlagDefaults() does not seem to be the first source. This is required to avoid data loss. If you really want to override data from previous sources, use FlagDefaultsDangerous()") + } + + return f(dst) + } +} + // FlagDefaultsDangerous obtains defaults from a flag Registerer and DANGEROUSLY // replaces (!!) the dst object with reg. // Make sure this is always the first source, or it will overwrite other data. diff --git a/pkg/cfg/flag_test.go b/pkg/cfg/flag_test.go index eac116fd5cce4..c3a821995f084 100644 --- a/pkg/cfg/flag_test.go +++ b/pkg/cfg/flag_test.go @@ -12,10 +12,10 @@ import ( // NOTE: These tests CANNOT run in parallel, because of the global state in the // `flag` package of the standard library. -// TestDefaults checks whether `FlagDefaultsDangerous()` correctly sets values from flag defaults +// TestDefaults checks whether `FlagDefaults()` correctly sets values from flag defaults func TestDefaults(t *testing.T) { var d Data - err := FlagDefaultsDangerous(&Data{}, nil)(&d) + err := FlagDefaults(&Data{}, nil)(&d) require.NoError(t, err) assert.Equal(t, Data{ Verbose: false, @@ -37,7 +37,7 @@ func TestFlagsSetOnly(t *testing.T) { var sharedMem []byte // c is not passed, to only see the output of dFlags() afterwards - err := FlagDefaultsDangerous(&Data{}, &sharedMem)(&Data{}) + err := FlagDefaults(&Data{}, &sharedMem)(&Data{}) require.NoError(t, err) err = dFlags([]string{"-verbose", "-server.timeout=12h"}, &Data{}, sharedMem)(&c) @@ -65,7 +65,7 @@ func TestFlagsMerge(t *testing.T) { var sharedMem []byte err := Unmarshal(&c, - FlagDefaultsDangerous(&Data{}, &sharedMem), + FlagDefaults(&Data{}, &sharedMem), dFlags([]string{"-verbose", "-server.timeout=12h"}, &Data{}, sharedMem), ) require.NoError(t, err) From d4d3804a39a35938a09a92e92795fc5d749aca61 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Tue, 17 Sep 2019 19:13:16 +0200 Subject: [PATCH 09/18] fix(loki): validation globals --- cmd/loki/main.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/loki/main.go b/cmd/loki/main.go index 750abb0e572a6..fd05500ad5b40 100644 --- a/cmd/loki/main.go +++ b/cmd/loki/main.go @@ -16,6 +16,7 @@ import ( "github.com/weaveworks/common/tracing" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/validation" ) func init() { @@ -34,6 +35,13 @@ func main() { os.Exit(0) } + // This global is set to the config passed into the last call to `NewOverrides`. If we don't + // call it atleast once, the defaults are set to an empty struct. + // We call it with the flag values so that the config file unmarshalling only overrides the values set in the config. + if _, err := validation.NewOverrides(config.LimitsConfig); err != nil { + log.Fatalln(err) + } + // Init the logger which will honor the log level set in config.Server if reflect.DeepEqual(&config.Server.LogLevel, &logging.Level{}) { level.Error(util.Logger).Log("msg", "invalid log level") From 79b2ccc91012a8a30a963ed299e9500a6fd5ab1a Mon Sep 17 00:00:00 2001 From: sh0rez Date: Mon, 23 Sep 2019 22:14:46 +0200 Subject: [PATCH 10/18] fix(pkg/cfg): distinguish set values from defaults Previously, the flag functionality relied on comparing archived defaults with the merged data after flag.Parse(). This inevitably led to a bug, that this package was able to decide whether a value is on default, or set by user to the same as default. This meant the user value was always lost. This now switches it to a more traditional usage of the flag package: 1. flag.***Var already sets the default value on the pointer 2. intermediate sources can make changes here 3. flag.Parse() ONLY sets new values --- cmd/promtail/main.go | 62 ++++++++++------------ pkg/cfg/cfg.go | 38 ++++++------- pkg/cfg/cfg_test.go | 6 +-- pkg/cfg/files.go | 4 +- pkg/cfg/flag.go | 124 +++++++++++++------------------------------ pkg/cfg/flag_test.go | 57 +------------------- 6 files changed, 88 insertions(+), 203 deletions(-) diff --git a/cmd/promtail/main.go b/cmd/promtail/main.go index 51ef9b4b2cf6d..d5c3ad4d81ef8 100644 --- a/cmd/promtail/main.go +++ b/cmd/promtail/main.go @@ -4,18 +4,12 @@ import ( "flag" "fmt" "log" - "os" - "reflect" - "github.com/cortexproject/cortex/pkg/util" - "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/version" - "github.com/weaveworks/common/logging" + "github.com/sanity-io/litter" "github.com/grafana/loki/pkg/cfg" - "github.com/grafana/loki/pkg/logentry/stages" - "github.com/grafana/loki/pkg/promtail" "github.com/grafana/loki/pkg/promtail/config" ) @@ -35,31 +29,33 @@ func main() { os.Exit(0) } - // Init the logger which will honor the log level set in cfg.Server - if reflect.DeepEqual(&config.ServerConfig.Config.LogLevel, &logging.Level{}) { - level.Error(util.Logger).Log("msg", "invalid log level") - os.Exit(1) - } - util.InitLogger(&config.ServerConfig.Config) - - // Set the global debug variable in the stages package which is used to conditionally log - // debug messages which otherwise cause huge allocations processing log lines for log messages never printed - if config.ServerConfig.Config.LogLevel.String() == "debug" { - stages.Debug = true - } - - p, err := promtail.New(config) - if err != nil { - level.Error(util.Logger).Log("msg", "error creating promtail", "error", err) - os.Exit(1) - } - - level.Info(util.Logger).Log("msg", "Starting Promtail", "version", version.Info()) + litter.Dump(config.ServerConfig) - if err := p.Run(); err != nil { - level.Error(util.Logger).Log("msg", "error starting promtail", "error", err) - os.Exit(1) - } - - p.Shutdown() + // Init the logger which will honor the log level set in cfg.Server + // if reflect.DeepEqual(&config.ServerConfig.Config.LogLevel, &logging.Level{}) { + // level.Error(util.Logger).Log("msg", "invalid log level") + // os.Exit(1) + // } + // util.InitLogger(&config.ServerConfig.Config) + + // // Set the global debug variable in the stages package which is used to conditionally log + // // debug messages which otherwise cause huge allocations processing log lines for log messages never printed + // if config.ServerConfig.Config.LogLevel.String() == "debug" { + // stages.Debug = true + // } + + // p, err := promtail.New(config) + // if err != nil { + // level.Error(util.Logger).Log("msg", "error creating promtail", "error", err) + // os.Exit(1) + // } + + // level.Info(util.Logger).Log("msg", "Starting Promtail", "version", version.Info()) + + // if err := p.Run(); err != nil { + // level.Error(util.Logger).Log("msg", "error starting promtail", "error", err) + // os.Exit(1) + // } + + // p.Shutdown() } diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go index 89a74ceb54373..2d3bb5e1444f9 100644 --- a/pkg/cfg/cfg.go +++ b/pkg/cfg/cfg.go @@ -1,9 +1,10 @@ package cfg import ( + "flag" + "os" "reflect" - "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/pkg/errors" ) @@ -38,19 +39,17 @@ func Unmarshal(dst interface{}, sources ...Source) error { // Parse is a higher level wrapper for Unmarshal that automatically parses flags and a .yaml file func Parse(dst interface{}) error { - yamlSource := func() Source { - return YAMLFlag("config.file", "", ".yaml configuration file to parse") + yamls := func() Source { + return YAMLFlag("config.file", "", "yaml file to load") } - flagSource := func(reg flagext.Registerer, def []byte) Source { - return Flags(reg, def) - } - return dParse(dst, yamlSource, flagSource) + + return dParse(dst, yamls, Flags) } -// dParse is like Parse, but allows dependency injection +// dParse is the same as Parse, but with dependency injection for testing func dParse(dst interface{}, - yamlSource func() Source, - flagSource func(flagext.Registerer, []byte) Source, + yamls func() Source, + flags func(fs *flag.FlagSet) Source, ) error { // check dst is a pointer v := reflect.ValueOf(dst) @@ -58,20 +57,13 @@ func dParse(dst interface{}, return ErrNotPointer } - // obtain type of dst for cloning - t := reflect.Indirect(v).Type() - - // create new instances of dst's type for flags - d := reflect.New(t).Interface().(flagext.Registerer) - f := reflect.New(t).Interface().(flagext.Registerer) - - // shared state for FlagDefaultsDangerous and Flags - var defaultsYaml []byte + fs := flag.NewFlagSet(os.Args[0]+" (cfg/internal)", flag.ExitOnError) // unmarshal config - return Unmarshal(dst, - FlagDefaults(d, &defaultsYaml), - yamlSource(), - flagSource(f, defaultsYaml), + err := Unmarshal(dst, + Defaults(fs), + yamls(), + flags(fs), ) + return err } diff --git a/pkg/cfg/cfg_test.go b/pkg/cfg/cfg_test.go index 1626c0733c507..92601c4205665 100644 --- a/pkg/cfg/cfg_test.go +++ b/pkg/cfg/cfg_test.go @@ -1,10 +1,10 @@ package cfg import ( + "flag" "testing" "time" - "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/stretchr/testify/require" ) @@ -19,8 +19,8 @@ tls: `)) } - flagSource := func(reg flagext.Registerer, def []byte) Source { - return dFlags([]string{"-verbose", "-server.port=21"}, reg, def) + flagSource := func(fs *flag.FlagSet) Source { + return dFlags(fs, []string{"-verbose", "-server.port=21"}) } var c Data diff --git a/pkg/cfg/files.go b/pkg/cfg/files.go index ee74114d7c564..c3f7308350a26 100644 --- a/pkg/cfg/files.go +++ b/pkg/cfg/files.go @@ -53,8 +53,10 @@ func dYAML(y []byte) Source { // YAMLFlag defines a `config.file` flag and loads this file func YAMLFlag(name, value, help string) Source { - f := flag.String(name, value, help) return func(dst interface{}) error { + f := flag.String(name, value, help) + flag.Parse() + if *f == "" { f = nil } diff --git a/pkg/cfg/flag.go b/pkg/cfg/flag.go index 636370d52488c..6523cbc571f38 100644 --- a/pkg/cfg/flag.go +++ b/pkg/cfg/flag.go @@ -3,113 +3,63 @@ package cfg import ( "flag" "os" - "reflect" "github.com/cortexproject/cortex/pkg/util/flagext" - ghodss "github.com/ghodss/yaml" - "github.com/pkg/errors" - yaml "gopkg.in/yaml.v2" ) -// FlagDefaults obtains defaults from a flag registerer and fails when it is not the first source. -func FlagDefaults(reg flagext.Registerer, ptr *[]byte) Source { - f := FlagDefaultsDangerous(reg, ptr) - +// Defaults registers fs (shared with Flags()) with dst as the +// flagext.Registerer Flags are also copied read-only to the global flag-set so +// that intermediate uses of flag.Parse() work just fine +func Defaults(fs *flag.FlagSet) Source { return func(dst interface{}) error { - // check dst is it's zero value (also indirect pointers first) - v := reflect.Indirect(reflect.ValueOf(dst)) - z := reflect.Zero(v.Type()) - if !reflect.DeepEqual(v.Interface(), z.Interface()) { - return errors.New("FlagDefaults() does not seem to be the first source. This is required to avoid data loss. If you really want to override data from previous sources, use FlagDefaultsDangerous()") + r, ok := dst.(flagext.Registerer) + if !ok { + panic("no") } - return f(dst) - } -} - -// FlagDefaultsDangerous obtains defaults from a flag Registerer and DANGEROUSLY -// replaces (!!) the dst object with reg. -// Make sure this is always the first source, or it will overwrite other data. -// The defaults are set into ptr, if is an empty byte slice and not nil. -func FlagDefaultsDangerous(reg flagext.Registerer, ptr *[]byte) Source { - // some cortex code uses global `flag.*Var`. To avoid the flag package panicking, the default flagset is temporarily swapped out - tmp := flag.CommandLine - flag.CommandLine = flag.NewFlagSet("", flag.ContinueOnError) - flagext.DefaultValues(reg) - flag.CommandLine = tmp - - // dump defaults for subsequent `Flags()` call - if ptr != nil { - d, err := yaml.Marshal(reg) - if err != nil { - panic(err) - } + // already sets the defaults on r + r.RegisterFlags(fs) - *ptr = d - } + // copy it all to the global flagset for help messages + flag.CommandLine = flag.NewFlagSet(os.Args[0]+" (cfg/global)", flag.ExitOnError) - return func(dst interface{}) error { - // set *reg for *dst - v := reflect.Indirect(reflect.ValueOf(dst)) - v.Set(reflect.Indirect(reflect.ValueOf(reg))) + fs.VisitAll(func(f *flag.Flag) { + flag.Var(discardValue{f.Value.String()}, f.Name, f.Usage) + }) return nil } } -// Flags returns a source that sets values from changed (!) flags onto dst. -// The must have the same structure as dst, usually it is even the same type. -// The defaults slice MUST contain valid defaults as yaml from the same registerer. -func Flags(reg flagext.Registerer, defaults []byte) Source { - return dFlags(os.Args[1:], reg, defaults) +// Flags parses the internal flagset, applying all values set on the command line +func Flags(fs *flag.FlagSet) Source { + return dFlags(fs, os.Args[1:]) } -// dFlags parses flags defined by reg using the args slice onto dst. -// The defaults slice MUST contain valid defaults as yaml from the same registerer. -func dFlags(args []string, reg flagext.Registerer, defaults []byte) Source { - flagext.RegisterFlags(reg) - _ = flag.CommandLine.Parse(args) - data, err := yaml.Marshal(reg) - if err != nil { - panic(err) - } +// dFlags parses the internal flagset, applying all values set on slice +func dFlags(fs *flag.FlagSet, args []string) Source { return func(dst interface{}) error { - return stripDefaults(dst, data, defaults) + // copy everything intermediate from the global set for help messages + flag.VisitAll(func(f *flag.Flag) { + if fs.Lookup(f.Name) == nil { + fs.Var(discardValue{f.Value.String()}, f.Name, f.Usage) + } + }) + + // parse the final flagset + fs.Parse(args) + return nil } } -// stripDefaults takes the final config, the config from flags and the defaults, -// but merges only non-default values onto dst -func stripDefaults(dst interface{}, flagsStr, defStr []byte) error { - // using ghodss.Unmarshal to receive map[string]interface{} - var live, def map[string]interface{} - if err := ghodss.Unmarshal(flagsStr, &live); err != nil { - return err - } - if err := ghodss.Unmarshal(defStr, &def); err != nil { - return err - } - - res := deleteSame(live, def, "") +// discardValue implements flag.Value, but discards any changes This is required +// when you want a FlagSet to include helps from another one, without affecting +// the actual values +type discardValue struct{ def string } - out, err := yaml.Marshal(res) - if err != nil { - return err - } - return yaml.Unmarshal(out, dst) +func (d discardValue) String() string { + return d.def } -// deleteSame filters all equal values out of deeply nested map[string]interface{}'s -func deleteSame(live, def map[string]interface{}, path string) map[string]interface{} { - for k := range live { - if reflect.DeepEqual(live[k], def[k]) { - delete(live, k) - continue - } - - if _, ok := live[k].(map[string]interface{}); ok { - live[k] = deleteSame(live[k].(map[string]interface{}), def[k].(map[string]interface{}), path+"."+k) - } - - } - return live +func (d discardValue) Set(string) error { + return nil } diff --git a/pkg/cfg/flag_test.go b/pkg/cfg/flag_test.go index c3a821995f084..7453cd6db8a59 100644 --- a/pkg/cfg/flag_test.go +++ b/pkg/cfg/flag_test.go @@ -9,13 +9,10 @@ import ( "github.com/stretchr/testify/require" ) -// NOTE: These tests CANNOT run in parallel, because of the global state in the -// `flag` package of the standard library. - // TestDefaults checks whether `FlagDefaults()` correctly sets values from flag defaults func TestDefaults(t *testing.T) { var d Data - err := FlagDefaults(&Data{}, nil)(&d) + err := Defaults(&flag.FlagSet{})(&d) require.NoError(t, err) assert.Equal(t, Data{ Verbose: false, @@ -29,55 +26,3 @@ func TestDefaults(t *testing.T) { }, }, d) } - -// TestFlagsSetOnly checks that user-supplied flag values can be correctly distinguished from defaults -func TestFlagsSetOnly(t *testing.T) { - flag.CommandLine = &flag.FlagSet{} - var c Data - - var sharedMem []byte - // c is not passed, to only see the output of dFlags() afterwards - err := FlagDefaults(&Data{}, &sharedMem)(&Data{}) - require.NoError(t, err) - - err = dFlags([]string{"-verbose", "-server.timeout=12h"}, &Data{}, sharedMem)(&c) - require.NoError(t, err) - - // check that defaults are correctly stripped away - assert.Equal(t, Data{ - Verbose: true, - Server: Server{ - Port: 0, - Timeout: 12 * time.Hour, - }, - TLS: TLS{ - Cert: "", - Key: "", - }, - }, c) -} - -// TestFlagsMerge checks that defaults and user-supplied values merge correctly -func TestFlagsMerge(t *testing.T) { - flag.CommandLine = &flag.FlagSet{} - - var c Data - var sharedMem []byte - - err := Unmarshal(&c, - FlagDefaults(&Data{}, &sharedMem), - dFlags([]string{"-verbose", "-server.timeout=12h"}, &Data{}, sharedMem), - ) - require.NoError(t, err) - assert.Equal(t, Data{ - Verbose: true, - Server: Server{ - Port: 80, - Timeout: 12 * time.Hour, - }, - TLS: TLS{ - Cert: "CERT", - Key: "KEY", - }, - }, c) -} From 16e43ae7780d7d4248509bb7b766b26638893a40 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Mon, 23 Sep 2019 22:18:15 +0200 Subject: [PATCH 11/18] Revert "chore(vendoring): ghodss/yaml, kelseyhightower/envconfig" This reverts commit 9b1b6d39ac00be9e992fd35f371c7c64b005719b. --- Gopkg.lock | 11 - vendor/github.com/ghodss/yaml/.gitignore | 20 - vendor/github.com/ghodss/yaml/.travis.yml | 7 - vendor/github.com/ghodss/yaml/LICENSE | 50 --- vendor/github.com/ghodss/yaml/README.md | 121 ------ vendor/github.com/ghodss/yaml/fields.go | 501 ---------------------- vendor/github.com/ghodss/yaml/yaml.go | 277 ------------ 7 files changed, 987 deletions(-) delete mode 100644 vendor/github.com/ghodss/yaml/.gitignore delete mode 100644 vendor/github.com/ghodss/yaml/.travis.yml delete mode 100644 vendor/github.com/ghodss/yaml/LICENSE delete mode 100644 vendor/github.com/ghodss/yaml/README.md delete mode 100644 vendor/github.com/ghodss/yaml/fields.go delete mode 100644 vendor/github.com/ghodss/yaml/yaml.go diff --git a/Gopkg.lock b/Gopkg.lock index 09fc44d8d13c5..c029adf7ee985 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -423,14 +423,6 @@ pruneopts = "UT" revision = "b364c791f57ac8f224c2f42ad599afa45ca0cfda" -[[projects]] - digest = "1:2cd7915ab26ede7d95b8749e6b1f933f1c6d5398030684e6505940a10f31cfda" - name = "github.com/ghodss/yaml" - packages = ["."] - pruneopts = "UT" - revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" - version = "v1.0.0" - [[projects]] digest = "1:ce058ca1b1787a997e578c4344f9650becb5086a3512332cbedffca9950d77a1" name = "github.com/go-kit/kit" @@ -1833,8 +1825,6 @@ "github.com/cortexproject/cortex/pkg/ingester/client", "github.com/cortexproject/cortex/pkg/ingester/index", "github.com/cortexproject/cortex/pkg/ring", - "github.com/cortexproject/cortex/pkg/ring/kv", - "github.com/cortexproject/cortex/pkg/ring/kv/codec", "github.com/cortexproject/cortex/pkg/util", "github.com/cortexproject/cortex/pkg/util/flagext", "github.com/cortexproject/cortex/pkg/util/grpcclient", @@ -1849,7 +1839,6 @@ "github.com/docker/docker/pkg/ioutils", "github.com/docker/go-plugins-helpers/sdk", "github.com/fatih/color", - "github.com/ghodss/yaml", "github.com/go-kit/kit/log", "github.com/go-kit/kit/log/level", "github.com/gogo/protobuf/gogoproto", diff --git a/vendor/github.com/ghodss/yaml/.gitignore b/vendor/github.com/ghodss/yaml/.gitignore deleted file mode 100644 index e256a31e00a52..0000000000000 --- a/vendor/github.com/ghodss/yaml/.gitignore +++ /dev/null @@ -1,20 +0,0 @@ -# OSX leaves these everywhere on SMB shares -._* - -# Eclipse files -.classpath -.project -.settings/** - -# Emacs save files -*~ - -# Vim-related files -[._]*.s[a-w][a-z] -[._]s[a-w][a-z] -*.un~ -Session.vim -.netrwhist - -# Go test binaries -*.test diff --git a/vendor/github.com/ghodss/yaml/.travis.yml b/vendor/github.com/ghodss/yaml/.travis.yml deleted file mode 100644 index 0e9d6edc010a6..0000000000000 --- a/vendor/github.com/ghodss/yaml/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: go -go: - - 1.3 - - 1.4 -script: - - go test - - go build diff --git a/vendor/github.com/ghodss/yaml/LICENSE b/vendor/github.com/ghodss/yaml/LICENSE deleted file mode 100644 index 7805d36de7305..0000000000000 --- a/vendor/github.com/ghodss/yaml/LICENSE +++ /dev/null @@ -1,50 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Sam Ghods - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/ghodss/yaml/README.md b/vendor/github.com/ghodss/yaml/README.md deleted file mode 100644 index 0200f75b4d126..0000000000000 --- a/vendor/github.com/ghodss/yaml/README.md +++ /dev/null @@ -1,121 +0,0 @@ -# YAML marshaling and unmarshaling support for Go - -[![Build Status](https://travis-ci.org/ghodss/yaml.svg)](https://travis-ci.org/ghodss/yaml) - -## Introduction - -A wrapper around [go-yaml](https://github.com/go-yaml/yaml) designed to enable a better way of handling YAML when marshaling to and from structs. - -In short, this library first converts YAML to JSON using go-yaml and then uses `json.Marshal` and `json.Unmarshal` to convert to or from the struct. This means that it effectively reuses the JSON struct tags as well as the custom JSON methods `MarshalJSON` and `UnmarshalJSON` unlike go-yaml. For a detailed overview of the rationale behind this method, [see this blog post](http://ghodss.com/2014/the-right-way-to-handle-yaml-in-golang/). - -## Compatibility - -This package uses [go-yaml](https://github.com/go-yaml/yaml) and therefore supports [everything go-yaml supports](https://github.com/go-yaml/yaml#compatibility). - -## Caveats - -**Caveat #1:** When using `yaml.Marshal` and `yaml.Unmarshal`, binary data should NOT be preceded with the `!!binary` YAML tag. If you do, go-yaml will convert the binary data from base64 to native binary data, which is not compatible with JSON. You can still use binary in your YAML files though - just store them without the `!!binary` tag and decode the base64 in your code (e.g. in the custom JSON methods `MarshalJSON` and `UnmarshalJSON`). This also has the benefit that your YAML and your JSON binary data will be decoded exactly the same way. As an example: - -``` -BAD: - exampleKey: !!binary gIGC - -GOOD: - exampleKey: gIGC -... and decode the base64 data in your code. -``` - -**Caveat #2:** When using `YAMLToJSON` directly, maps with keys that are maps will result in an error since this is not supported by JSON. This error will occur in `Unmarshal` as well since you can't unmarshal map keys anyways since struct fields can't be keys. - -## Installation and usage - -To install, run: - -``` -$ go get github.com/ghodss/yaml -``` - -And import using: - -``` -import "github.com/ghodss/yaml" -``` - -Usage is very similar to the JSON library: - -```go -package main - -import ( - "fmt" - - "github.com/ghodss/yaml" -) - -type Person struct { - Name string `json:"name"` // Affects YAML field names too. - Age int `json:"age"` -} - -func main() { - // Marshal a Person struct to YAML. - p := Person{"John", 30} - y, err := yaml.Marshal(p) - if err != nil { - fmt.Printf("err: %v\n", err) - return - } - fmt.Println(string(y)) - /* Output: - age: 30 - name: John - */ - - // Unmarshal the YAML back into a Person struct. - var p2 Person - err = yaml.Unmarshal(y, &p2) - if err != nil { - fmt.Printf("err: %v\n", err) - return - } - fmt.Println(p2) - /* Output: - {John 30} - */ -} -``` - -`yaml.YAMLToJSON` and `yaml.JSONToYAML` methods are also available: - -```go -package main - -import ( - "fmt" - - "github.com/ghodss/yaml" -) - -func main() { - j := []byte(`{"name": "John", "age": 30}`) - y, err := yaml.JSONToYAML(j) - if err != nil { - fmt.Printf("err: %v\n", err) - return - } - fmt.Println(string(y)) - /* Output: - name: John - age: 30 - */ - j2, err := yaml.YAMLToJSON(y) - if err != nil { - fmt.Printf("err: %v\n", err) - return - } - fmt.Println(string(j2)) - /* Output: - {"age":30,"name":"John"} - */ -} -``` diff --git a/vendor/github.com/ghodss/yaml/fields.go b/vendor/github.com/ghodss/yaml/fields.go deleted file mode 100644 index 58600740266c6..0000000000000 --- a/vendor/github.com/ghodss/yaml/fields.go +++ /dev/null @@ -1,501 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -package yaml - -import ( - "bytes" - "encoding" - "encoding/json" - "reflect" - "sort" - "strings" - "sync" - "unicode" - "unicode/utf8" -) - -// indirect walks down v allocating pointers as needed, -// until it gets to a non-pointer. -// if it encounters an Unmarshaler, indirect stops and returns that. -// if decodingNull is true, indirect stops at the last pointer so it can be set to nil. -func indirect(v reflect.Value, decodingNull bool) (json.Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { - // If v is a named type and is addressable, - // start with its address, so that if the type has pointer methods, - // we find them. - if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { - v = v.Addr() - } - for { - // Load value from interface, but only if the result will be - // usefully addressable. - if v.Kind() == reflect.Interface && !v.IsNil() { - e := v.Elem() - if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { - v = e - continue - } - } - - if v.Kind() != reflect.Ptr { - break - } - - if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() { - break - } - if v.IsNil() { - if v.CanSet() { - v.Set(reflect.New(v.Type().Elem())) - } else { - v = reflect.New(v.Type().Elem()) - } - } - if v.Type().NumMethod() > 0 { - if u, ok := v.Interface().(json.Unmarshaler); ok { - return u, nil, reflect.Value{} - } - if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { - return nil, u, reflect.Value{} - } - } - v = v.Elem() - } - return nil, nil, v -} - -// A field represents a single field found in a struct. -type field struct { - name string - nameBytes []byte // []byte(name) - equalFold func(s, t []byte) bool // bytes.EqualFold or equivalent - - tag bool - index []int - typ reflect.Type - omitEmpty bool - quoted bool -} - -func fillField(f field) field { - f.nameBytes = []byte(f.name) - f.equalFold = foldFunc(f.nameBytes) - return f -} - -// byName sorts field by name, breaking ties with depth, -// then breaking ties with "name came from json tag", then -// breaking ties with index sequence. -type byName []field - -func (x byName) Len() int { return len(x) } - -func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - -func (x byName) Less(i, j int) bool { - if x[i].name != x[j].name { - return x[i].name < x[j].name - } - if len(x[i].index) != len(x[j].index) { - return len(x[i].index) < len(x[j].index) - } - if x[i].tag != x[j].tag { - return x[i].tag - } - return byIndex(x).Less(i, j) -} - -// byIndex sorts field by index sequence. -type byIndex []field - -func (x byIndex) Len() int { return len(x) } - -func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - -func (x byIndex) Less(i, j int) bool { - for k, xik := range x[i].index { - if k >= len(x[j].index) { - return false - } - if xik != x[j].index[k] { - return xik < x[j].index[k] - } - } - return len(x[i].index) < len(x[j].index) -} - -// typeFields returns a list of fields that JSON should recognize for the given type. -// The algorithm is breadth-first search over the set of structs to include - the top struct -// and then any reachable anonymous structs. -func typeFields(t reflect.Type) []field { - // Anonymous fields to explore at the current level and the next. - current := []field{} - next := []field{{typ: t}} - - // Count of queued names for current level and the next. - count := map[reflect.Type]int{} - nextCount := map[reflect.Type]int{} - - // Types already visited at an earlier level. - visited := map[reflect.Type]bool{} - - // Fields found. - var fields []field - - for len(next) > 0 { - current, next = next, current[:0] - count, nextCount = nextCount, map[reflect.Type]int{} - - for _, f := range current { - if visited[f.typ] { - continue - } - visited[f.typ] = true - - // Scan f.typ for fields to include. - for i := 0; i < f.typ.NumField(); i++ { - sf := f.typ.Field(i) - if sf.PkgPath != "" { // unexported - continue - } - tag := sf.Tag.Get("json") - if tag == "-" { - continue - } - name, opts := parseTag(tag) - if !isValidTag(name) { - name = "" - } - index := make([]int, len(f.index)+1) - copy(index, f.index) - index[len(f.index)] = i - - ft := sf.Type - if ft.Name() == "" && ft.Kind() == reflect.Ptr { - // Follow pointer. - ft = ft.Elem() - } - - // Record found field and index sequence. - if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { - tagged := name != "" - if name == "" { - name = sf.Name - } - fields = append(fields, fillField(field{ - name: name, - tag: tagged, - index: index, - typ: ft, - omitEmpty: opts.Contains("omitempty"), - quoted: opts.Contains("string"), - })) - if count[f.typ] > 1 { - // If there were multiple instances, add a second, - // so that the annihilation code will see a duplicate. - // It only cares about the distinction between 1 or 2, - // so don't bother generating any more copies. - fields = append(fields, fields[len(fields)-1]) - } - continue - } - - // Record new anonymous struct to explore in next round. - nextCount[ft]++ - if nextCount[ft] == 1 { - next = append(next, fillField(field{name: ft.Name(), index: index, typ: ft})) - } - } - } - } - - sort.Sort(byName(fields)) - - // Delete all fields that are hidden by the Go rules for embedded fields, - // except that fields with JSON tags are promoted. - - // The fields are sorted in primary order of name, secondary order - // of field index length. Loop over names; for each name, delete - // hidden fields by choosing the one dominant field that survives. - out := fields[:0] - for advance, i := 0, 0; i < len(fields); i += advance { - // One iteration per name. - // Find the sequence of fields with the name of this first field. - fi := fields[i] - name := fi.name - for advance = 1; i+advance < len(fields); advance++ { - fj := fields[i+advance] - if fj.name != name { - break - } - } - if advance == 1 { // Only one field with this name - out = append(out, fi) - continue - } - dominant, ok := dominantField(fields[i : i+advance]) - if ok { - out = append(out, dominant) - } - } - - fields = out - sort.Sort(byIndex(fields)) - - return fields -} - -// dominantField looks through the fields, all of which are known to -// have the same name, to find the single field that dominates the -// others using Go's embedding rules, modified by the presence of -// JSON tags. If there are multiple top-level fields, the boolean -// will be false: This condition is an error in Go and we skip all -// the fields. -func dominantField(fields []field) (field, bool) { - // The fields are sorted in increasing index-length order. The winner - // must therefore be one with the shortest index length. Drop all - // longer entries, which is easy: just truncate the slice. - length := len(fields[0].index) - tagged := -1 // Index of first tagged field. - for i, f := range fields { - if len(f.index) > length { - fields = fields[:i] - break - } - if f.tag { - if tagged >= 0 { - // Multiple tagged fields at the same level: conflict. - // Return no field. - return field{}, false - } - tagged = i - } - } - if tagged >= 0 { - return fields[tagged], true - } - // All remaining fields have the same length. If there's more than one, - // we have a conflict (two fields named "X" at the same level) and we - // return no field. - if len(fields) > 1 { - return field{}, false - } - return fields[0], true -} - -var fieldCache struct { - sync.RWMutex - m map[reflect.Type][]field -} - -// cachedTypeFields is like typeFields but uses a cache to avoid repeated work. -func cachedTypeFields(t reflect.Type) []field { - fieldCache.RLock() - f := fieldCache.m[t] - fieldCache.RUnlock() - if f != nil { - return f - } - - // Compute fields without lock. - // Might duplicate effort but won't hold other computations back. - f = typeFields(t) - if f == nil { - f = []field{} - } - - fieldCache.Lock() - if fieldCache.m == nil { - fieldCache.m = map[reflect.Type][]field{} - } - fieldCache.m[t] = f - fieldCache.Unlock() - return f -} - -func isValidTag(s string) bool { - if s == "" { - return false - } - for _, c := range s { - switch { - case strings.ContainsRune("!#$%&()*+-./:<=>?@[]^_{|}~ ", c): - // Backslash and quote chars are reserved, but - // otherwise any punctuation chars are allowed - // in a tag name. - default: - if !unicode.IsLetter(c) && !unicode.IsDigit(c) { - return false - } - } - } - return true -} - -const ( - caseMask = ^byte(0x20) // Mask to ignore case in ASCII. - kelvin = '\u212a' - smallLongEss = '\u017f' -) - -// foldFunc returns one of four different case folding equivalence -// functions, from most general (and slow) to fastest: -// -// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8 -// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S') -// 3) asciiEqualFold, no special, but includes non-letters (including _) -// 4) simpleLetterEqualFold, no specials, no non-letters. -// -// The letters S and K are special because they map to 3 runes, not just 2: -// * S maps to s and to U+017F 'ſ' Latin small letter long s -// * k maps to K and to U+212A 'K' Kelvin sign -// See http://play.golang.org/p/tTxjOc0OGo -// -// The returned function is specialized for matching against s and -// should only be given s. It's not curried for performance reasons. -func foldFunc(s []byte) func(s, t []byte) bool { - nonLetter := false - special := false // special letter - for _, b := range s { - if b >= utf8.RuneSelf { - return bytes.EqualFold - } - upper := b & caseMask - if upper < 'A' || upper > 'Z' { - nonLetter = true - } else if upper == 'K' || upper == 'S' { - // See above for why these letters are special. - special = true - } - } - if special { - return equalFoldRight - } - if nonLetter { - return asciiEqualFold - } - return simpleLetterEqualFold -} - -// equalFoldRight is a specialization of bytes.EqualFold when s is -// known to be all ASCII (including punctuation), but contains an 's', -// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t. -// See comments on foldFunc. -func equalFoldRight(s, t []byte) bool { - for _, sb := range s { - if len(t) == 0 { - return false - } - tb := t[0] - if tb < utf8.RuneSelf { - if sb != tb { - sbUpper := sb & caseMask - if 'A' <= sbUpper && sbUpper <= 'Z' { - if sbUpper != tb&caseMask { - return false - } - } else { - return false - } - } - t = t[1:] - continue - } - // sb is ASCII and t is not. t must be either kelvin - // sign or long s; sb must be s, S, k, or K. - tr, size := utf8.DecodeRune(t) - switch sb { - case 's', 'S': - if tr != smallLongEss { - return false - } - case 'k', 'K': - if tr != kelvin { - return false - } - default: - return false - } - t = t[size:] - - } - if len(t) > 0 { - return false - } - return true -} - -// asciiEqualFold is a specialization of bytes.EqualFold for use when -// s is all ASCII (but may contain non-letters) and contains no -// special-folding letters. -// See comments on foldFunc. -func asciiEqualFold(s, t []byte) bool { - if len(s) != len(t) { - return false - } - for i, sb := range s { - tb := t[i] - if sb == tb { - continue - } - if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') { - if sb&caseMask != tb&caseMask { - return false - } - } else { - return false - } - } - return true -} - -// simpleLetterEqualFold is a specialization of bytes.EqualFold for -// use when s is all ASCII letters (no underscores, etc) and also -// doesn't contain 'k', 'K', 's', or 'S'. -// See comments on foldFunc. -func simpleLetterEqualFold(s, t []byte) bool { - if len(s) != len(t) { - return false - } - for i, b := range s { - if b&caseMask != t[i]&caseMask { - return false - } - } - return true -} - -// tagOptions is the string following a comma in a struct field's "json" -// tag, or the empty string. It does not include the leading comma. -type tagOptions string - -// parseTag splits a struct field's json tag into its name and -// comma-separated options. -func parseTag(tag string) (string, tagOptions) { - if idx := strings.Index(tag, ","); idx != -1 { - return tag[:idx], tagOptions(tag[idx+1:]) - } - return tag, tagOptions("") -} - -// Contains reports whether a comma-separated list of options -// contains a particular substr flag. substr must be surrounded by a -// string boundary or commas. -func (o tagOptions) Contains(optionName string) bool { - if len(o) == 0 { - return false - } - s := string(o) - for s != "" { - var next string - i := strings.Index(s, ",") - if i >= 0 { - s, next = s[:i], s[i+1:] - } - if s == optionName { - return true - } - s = next - } - return false -} diff --git a/vendor/github.com/ghodss/yaml/yaml.go b/vendor/github.com/ghodss/yaml/yaml.go deleted file mode 100644 index 4fb4054a8b747..0000000000000 --- a/vendor/github.com/ghodss/yaml/yaml.go +++ /dev/null @@ -1,277 +0,0 @@ -package yaml - -import ( - "bytes" - "encoding/json" - "fmt" - "reflect" - "strconv" - - "gopkg.in/yaml.v2" -) - -// Marshals the object into JSON then converts JSON to YAML and returns the -// YAML. -func Marshal(o interface{}) ([]byte, error) { - j, err := json.Marshal(o) - if err != nil { - return nil, fmt.Errorf("error marshaling into JSON: %v", err) - } - - y, err := JSONToYAML(j) - if err != nil { - return nil, fmt.Errorf("error converting JSON to YAML: %v", err) - } - - return y, nil -} - -// Converts YAML to JSON then uses JSON to unmarshal into an object. -func Unmarshal(y []byte, o interface{}) error { - vo := reflect.ValueOf(o) - j, err := yamlToJSON(y, &vo) - if err != nil { - return fmt.Errorf("error converting YAML to JSON: %v", err) - } - - err = json.Unmarshal(j, o) - if err != nil { - return fmt.Errorf("error unmarshaling JSON: %v", err) - } - - return nil -} - -// Convert JSON to YAML. -func JSONToYAML(j []byte) ([]byte, error) { - // Convert the JSON to an object. - var jsonObj interface{} - // We are using yaml.Unmarshal here (instead of json.Unmarshal) because the - // Go JSON library doesn't try to pick the right number type (int, float, - // etc.) when unmarshalling to interface{}, it just picks float64 - // universally. go-yaml does go through the effort of picking the right - // number type, so we can preserve number type throughout this process. - err := yaml.Unmarshal(j, &jsonObj) - if err != nil { - return nil, err - } - - // Marshal this object into YAML. - return yaml.Marshal(jsonObj) -} - -// Convert YAML to JSON. Since JSON is a subset of YAML, passing JSON through -// this method should be a no-op. -// -// Things YAML can do that are not supported by JSON: -// * In YAML you can have binary and null keys in your maps. These are invalid -// in JSON. (int and float keys are converted to strings.) -// * Binary data in YAML with the !!binary tag is not supported. If you want to -// use binary data with this library, encode the data as base64 as usual but do -// not use the !!binary tag in your YAML. This will ensure the original base64 -// encoded data makes it all the way through to the JSON. -func YAMLToJSON(y []byte) ([]byte, error) { - return yamlToJSON(y, nil) -} - -func yamlToJSON(y []byte, jsonTarget *reflect.Value) ([]byte, error) { - // Convert the YAML to an object. - var yamlObj interface{} - err := yaml.Unmarshal(y, &yamlObj) - if err != nil { - return nil, err - } - - // YAML objects are not completely compatible with JSON objects (e.g. you - // can have non-string keys in YAML). So, convert the YAML-compatible object - // to a JSON-compatible object, failing with an error if irrecoverable - // incompatibilties happen along the way. - jsonObj, err := convertToJSONableObject(yamlObj, jsonTarget) - if err != nil { - return nil, err - } - - // Convert this object to JSON and return the data. - return json.Marshal(jsonObj) -} - -func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (interface{}, error) { - var err error - - // Resolve jsonTarget to a concrete value (i.e. not a pointer or an - // interface). We pass decodingNull as false because we're not actually - // decoding into the value, we're just checking if the ultimate target is a - // string. - if jsonTarget != nil { - ju, tu, pv := indirect(*jsonTarget, false) - // We have a JSON or Text Umarshaler at this level, so we can't be trying - // to decode into a string. - if ju != nil || tu != nil { - jsonTarget = nil - } else { - jsonTarget = &pv - } - } - - // If yamlObj is a number or a boolean, check if jsonTarget is a string - - // if so, coerce. Else return normal. - // If yamlObj is a map or array, find the field that each key is - // unmarshaling to, and when you recurse pass the reflect.Value for that - // field back into this function. - switch typedYAMLObj := yamlObj.(type) { - case map[interface{}]interface{}: - // JSON does not support arbitrary keys in a map, so we must convert - // these keys to strings. - // - // From my reading of go-yaml v2 (specifically the resolve function), - // keys can only have the types string, int, int64, float64, binary - // (unsupported), or null (unsupported). - strMap := make(map[string]interface{}) - for k, v := range typedYAMLObj { - // Resolve the key to a string first. - var keyString string - switch typedKey := k.(type) { - case string: - keyString = typedKey - case int: - keyString = strconv.Itoa(typedKey) - case int64: - // go-yaml will only return an int64 as a key if the system - // architecture is 32-bit and the key's value is between 32-bit - // and 64-bit. Otherwise the key type will simply be int. - keyString = strconv.FormatInt(typedKey, 10) - case float64: - // Stolen from go-yaml to use the same conversion to string as - // the go-yaml library uses to convert float to string when - // Marshaling. - s := strconv.FormatFloat(typedKey, 'g', -1, 32) - switch s { - case "+Inf": - s = ".inf" - case "-Inf": - s = "-.inf" - case "NaN": - s = ".nan" - } - keyString = s - case bool: - if typedKey { - keyString = "true" - } else { - keyString = "false" - } - default: - return nil, fmt.Errorf("Unsupported map key of type: %s, key: %+#v, value: %+#v", - reflect.TypeOf(k), k, v) - } - - // jsonTarget should be a struct or a map. If it's a struct, find - // the field it's going to map to and pass its reflect.Value. If - // it's a map, find the element type of the map and pass the - // reflect.Value created from that type. If it's neither, just pass - // nil - JSON conversion will error for us if it's a real issue. - if jsonTarget != nil { - t := *jsonTarget - if t.Kind() == reflect.Struct { - keyBytes := []byte(keyString) - // Find the field that the JSON library would use. - var f *field - fields := cachedTypeFields(t.Type()) - for i := range fields { - ff := &fields[i] - if bytes.Equal(ff.nameBytes, keyBytes) { - f = ff - break - } - // Do case-insensitive comparison. - if f == nil && ff.equalFold(ff.nameBytes, keyBytes) { - f = ff - } - } - if f != nil { - // Find the reflect.Value of the most preferential - // struct field. - jtf := t.Field(f.index[0]) - strMap[keyString], err = convertToJSONableObject(v, &jtf) - if err != nil { - return nil, err - } - continue - } - } else if t.Kind() == reflect.Map { - // Create a zero value of the map's element type to use as - // the JSON target. - jtv := reflect.Zero(t.Type().Elem()) - strMap[keyString], err = convertToJSONableObject(v, &jtv) - if err != nil { - return nil, err - } - continue - } - } - strMap[keyString], err = convertToJSONableObject(v, nil) - if err != nil { - return nil, err - } - } - return strMap, nil - case []interface{}: - // We need to recurse into arrays in case there are any - // map[interface{}]interface{}'s inside and to convert any - // numbers to strings. - - // If jsonTarget is a slice (which it really should be), find the - // thing it's going to map to. If it's not a slice, just pass nil - // - JSON conversion will error for us if it's a real issue. - var jsonSliceElemValue *reflect.Value - if jsonTarget != nil { - t := *jsonTarget - if t.Kind() == reflect.Slice { - // By default slices point to nil, but we need a reflect.Value - // pointing to a value of the slice type, so we create one here. - ev := reflect.Indirect(reflect.New(t.Type().Elem())) - jsonSliceElemValue = &ev - } - } - - // Make and use a new array. - arr := make([]interface{}, len(typedYAMLObj)) - for i, v := range typedYAMLObj { - arr[i], err = convertToJSONableObject(v, jsonSliceElemValue) - if err != nil { - return nil, err - } - } - return arr, nil - default: - // If the target type is a string and the YAML type is a number, - // convert the YAML type to a string. - if jsonTarget != nil && (*jsonTarget).Kind() == reflect.String { - // Based on my reading of go-yaml, it may return int, int64, - // float64, or uint64. - var s string - switch typedVal := typedYAMLObj.(type) { - case int: - s = strconv.FormatInt(int64(typedVal), 10) - case int64: - s = strconv.FormatInt(typedVal, 10) - case float64: - s = strconv.FormatFloat(typedVal, 'g', -1, 32) - case uint64: - s = strconv.FormatUint(typedVal, 10) - case bool: - if typedVal { - s = "true" - } else { - s = "false" - } - } - if len(s) > 0 { - yamlObj = interface{}(s) - } - } - return yamlObj, nil - } - - return nil, nil -} From 96805334edde0642aa558d8339985d6e1dd35b5b Mon Sep 17 00:00:00 2001 From: sh0rez Date: Mon, 23 Sep 2019 22:19:51 +0200 Subject: [PATCH 12/18] fix(cmd/promtail): remove debug code --- cmd/promtail/main.go | 62 +++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/cmd/promtail/main.go b/cmd/promtail/main.go index d5c3ad4d81ef8..51ef9b4b2cf6d 100644 --- a/cmd/promtail/main.go +++ b/cmd/promtail/main.go @@ -4,12 +4,18 @@ import ( "flag" "fmt" "log" + "os" + "reflect" + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/version" - "github.com/sanity-io/litter" + "github.com/weaveworks/common/logging" "github.com/grafana/loki/pkg/cfg" + "github.com/grafana/loki/pkg/logentry/stages" + "github.com/grafana/loki/pkg/promtail" "github.com/grafana/loki/pkg/promtail/config" ) @@ -29,33 +35,31 @@ func main() { os.Exit(0) } - litter.Dump(config.ServerConfig) - // Init the logger which will honor the log level set in cfg.Server - // if reflect.DeepEqual(&config.ServerConfig.Config.LogLevel, &logging.Level{}) { - // level.Error(util.Logger).Log("msg", "invalid log level") - // os.Exit(1) - // } - // util.InitLogger(&config.ServerConfig.Config) - - // // Set the global debug variable in the stages package which is used to conditionally log - // // debug messages which otherwise cause huge allocations processing log lines for log messages never printed - // if config.ServerConfig.Config.LogLevel.String() == "debug" { - // stages.Debug = true - // } - - // p, err := promtail.New(config) - // if err != nil { - // level.Error(util.Logger).Log("msg", "error creating promtail", "error", err) - // os.Exit(1) - // } - - // level.Info(util.Logger).Log("msg", "Starting Promtail", "version", version.Info()) - - // if err := p.Run(); err != nil { - // level.Error(util.Logger).Log("msg", "error starting promtail", "error", err) - // os.Exit(1) - // } - - // p.Shutdown() + if reflect.DeepEqual(&config.ServerConfig.Config.LogLevel, &logging.Level{}) { + level.Error(util.Logger).Log("msg", "invalid log level") + os.Exit(1) + } + util.InitLogger(&config.ServerConfig.Config) + + // Set the global debug variable in the stages package which is used to conditionally log + // debug messages which otherwise cause huge allocations processing log lines for log messages never printed + if config.ServerConfig.Config.LogLevel.String() == "debug" { + stages.Debug = true + } + + p, err := promtail.New(config) + if err != nil { + level.Error(util.Logger).Log("msg", "error creating promtail", "error", err) + os.Exit(1) + } + + level.Info(util.Logger).Log("msg", "Starting Promtail", "version", version.Info()) + + if err := p.Run(); err != nil { + level.Error(util.Logger).Log("msg", "error starting promtail", "error", err) + os.Exit(1) + } + + p.Shutdown() } From c3abcacf5bbfccd1317c5d653337a62053bec91e Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 26 Sep 2019 12:16:30 +0200 Subject: [PATCH 13/18] fix(cfg): unify flagsets --- cmd/loki/main.go | 2 +- pkg/cfg/cfg.go | 21 ++++++++----------- pkg/cfg/flag.go | 52 +++++++++++------------------------------------- 3 files changed, 21 insertions(+), 54 deletions(-) diff --git a/cmd/loki/main.go b/cmd/loki/main.go index fd05500ad5b40..a8b4a07bbebff 100644 --- a/cmd/loki/main.go +++ b/cmd/loki/main.go @@ -16,7 +16,7 @@ import ( "github.com/weaveworks/common/tracing" "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/validation" + "github.com/grafana/loki/pkg/util/validation" ) func init() { diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go index 2d3bb5e1444f9..cf29420931ab3 100644 --- a/pkg/cfg/cfg.go +++ b/pkg/cfg/cfg.go @@ -1,8 +1,6 @@ package cfg import ( - "flag" - "os" "reflect" "github.com/pkg/errors" @@ -13,7 +11,10 @@ import ( // destination, which will be something compatible to `json.Unmarshal`. The // obtained configuration may be written to this object, it may also contain // data from previous sources. -type Source func(interface{}) error +type ( + Source func(interface{}) error + SourceFunc func() Source +) var ( ErrNotPointer = errors.New("dst is not a pointer") @@ -47,23 +48,17 @@ func Parse(dst interface{}) error { } // dParse is the same as Parse, but with dependency injection for testing -func dParse(dst interface{}, - yamls func() Source, - flags func(fs *flag.FlagSet) Source, -) error { +func dParse(dst interface{}, yamls, flags SourceFunc) error { // check dst is a pointer v := reflect.ValueOf(dst) if v.Kind() != reflect.Ptr { return ErrNotPointer } - fs := flag.NewFlagSet(os.Args[0]+" (cfg/internal)", flag.ExitOnError) - // unmarshal config - err := Unmarshal(dst, - Defaults(fs), + return Unmarshal(dst, + Defaults(), yamls(), - flags(fs), + flags(), ) - return err } diff --git a/pkg/cfg/flag.go b/pkg/cfg/flag.go index 6523cbc571f38..27b2c56bc43f4 100644 --- a/pkg/cfg/flag.go +++ b/pkg/cfg/flag.go @@ -5,61 +5,33 @@ import ( "os" "github.com/cortexproject/cortex/pkg/util/flagext" + "github.com/pkg/errors" ) -// Defaults registers fs (shared with Flags()) with dst as the -// flagext.Registerer Flags are also copied read-only to the global flag-set so -// that intermediate uses of flag.Parse() work just fine -func Defaults(fs *flag.FlagSet) Source { +// Defaults registers flags using dst as the flagext.Registerer +func Defaults() Source { return func(dst interface{}) error { r, ok := dst.(flagext.Registerer) if !ok { - panic("no") + return errors.New("dst does not satisfy flagext.Registerer") } // already sets the defaults on r - r.RegisterFlags(fs) - - // copy it all to the global flagset for help messages - flag.CommandLine = flag.NewFlagSet(os.Args[0]+" (cfg/global)", flag.ExitOnError) - - fs.VisitAll(func(f *flag.Flag) { - flag.Var(discardValue{f.Value.String()}, f.Name, f.Usage) - }) + r.RegisterFlags(flag.CommandLine) return nil } } -// Flags parses the internal flagset, applying all values set on the command line -func Flags(fs *flag.FlagSet) Source { - return dFlags(fs, os.Args[1:]) +// Flags parses the flag from the command line, setting only user-supplied +// values on the flagext.Registerer passed to Defaults() +func Flags() Source { + return dFlags(os.Args[1:]) } -// dFlags parses the internal flagset, applying all values set on slice -func dFlags(fs *flag.FlagSet, args []string) Source { +// dFlags parses the flagset, applying all values set on the slice +func dFlags(args []string) Source { return func(dst interface{}) error { - // copy everything intermediate from the global set for help messages - flag.VisitAll(func(f *flag.Flag) { - if fs.Lookup(f.Name) == nil { - fs.Var(discardValue{f.Value.String()}, f.Name, f.Usage) - } - }) - // parse the final flagset - fs.Parse(args) - return nil + return flag.CommandLine.Parse(args) } } - -// discardValue implements flag.Value, but discards any changes This is required -// when you want a FlagSet to include helps from another one, without affecting -// the actual values -type discardValue struct{ def string } - -func (d discardValue) String() string { - return d.def -} - -func (d discardValue) Set(string) error { - return nil -} From b9b28e27f9f50c207eca519e8d95096198375dfe Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 26 Sep 2019 12:39:51 +0200 Subject: [PATCH 14/18] test(cfg): adapt tests to recent changes --- pkg/cfg/cfg.go | 23 ++++++++++------------- pkg/cfg/cfg_test.go | 14 +++++--------- pkg/cfg/data_test.go | 5 +++-- pkg/cfg/flag.go | 16 +++++++++++----- pkg/cfg/flag_test.go | 42 ++++++++++++++++++++++++++++++++++++------ 5 files changed, 65 insertions(+), 35 deletions(-) diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go index cf29420931ab3..789c733a8075b 100644 --- a/pkg/cfg/cfg.go +++ b/pkg/cfg/cfg.go @@ -11,10 +11,7 @@ import ( // destination, which will be something compatible to `json.Unmarshal`. The // obtained configuration may be written to this object, it may also contain // data from previous sources. -type ( - Source func(interface{}) error - SourceFunc func() Source -) +type Source func(interface{}) error var ( ErrNotPointer = errors.New("dst is not a pointer") @@ -40,15 +37,15 @@ func Unmarshal(dst interface{}, sources ...Source) error { // Parse is a higher level wrapper for Unmarshal that automatically parses flags and a .yaml file func Parse(dst interface{}) error { - yamls := func() Source { - return YAMLFlag("config.file", "", "yaml file to load") - } - - return dParse(dst, yamls, Flags) + return dParse(dst, + Defaults(), + YAMLFlag("config.file", "", "yaml file to load"), + Flags(), + ) } // dParse is the same as Parse, but with dependency injection for testing -func dParse(dst interface{}, yamls, flags SourceFunc) error { +func dParse(dst interface{}, defaults, yaml, flags Source) error { // check dst is a pointer v := reflect.ValueOf(dst) if v.Kind() != reflect.Ptr { @@ -57,8 +54,8 @@ func dParse(dst interface{}, yamls, flags SourceFunc) error { // unmarshal config return Unmarshal(dst, - Defaults(), - yamls(), - flags(), + defaults, + yaml, + flags, ) } diff --git a/pkg/cfg/cfg_test.go b/pkg/cfg/cfg_test.go index 92601c4205665..87192f0de0290 100644 --- a/pkg/cfg/cfg_test.go +++ b/pkg/cfg/cfg_test.go @@ -9,22 +9,18 @@ import ( ) func TestParse(t *testing.T) { - yamlSource := func() Source { - return dYAML([]byte(` + yamlSource := dYAML([]byte(` server: port: 2000 timeout: 60h tls: key: YAML `)) - } - - flagSource := func(fs *flag.FlagSet) Source { - return dFlags(fs, []string{"-verbose", "-server.port=21"}) - } + fs := flag.NewFlagSet("testParse", flag.PanicOnError) + flagSource := dFlags(fs, []string{"-verbose", "-server.port=21"}) var c Data - err := dParse(&c, yamlSource, flagSource) + err := dParse(&c, dDefaults(fs), yamlSource, flagSource) require.NoError(t, err) require.Equal(t, Data{ @@ -34,7 +30,7 @@ tls: Timeout: 60 * time.Hour, }, TLS: TLS{ - Cert: "CERT", + Cert: "DEFAULTCERT", Key: "YAML", }, }, c) diff --git a/pkg/cfg/data_test.go b/pkg/cfg/data_test.go index 080886cf81ac6..b262c492b1a4b 100644 --- a/pkg/cfg/data_test.go +++ b/pkg/cfg/data_test.go @@ -22,11 +22,12 @@ type TLS struct { Key string `yaml:"key"` } +// RegisterFlags makes Data implement flagext.Registerer for using flags func (d *Data) RegisterFlags(fs *flag.FlagSet) { fs.BoolVar(&d.Verbose, "verbose", false, "") fs.IntVar(&d.Server.Port, "server.port", 80, "") fs.DurationVar(&d.Server.Timeout, "server.timeout", 60*time.Second, "") - fs.StringVar(&d.TLS.Cert, "tls.cert", "CERT", "") - fs.StringVar(&d.TLS.Key, "tls.key", "KEY", "") + fs.StringVar(&d.TLS.Cert, "tls.cert", "DEFAULTCERT", "") + fs.StringVar(&d.TLS.Key, "tls.key", "DEFAULTKEY", "") } diff --git a/pkg/cfg/flag.go b/pkg/cfg/flag.go index 27b2c56bc43f4..b895a7b408685 100644 --- a/pkg/cfg/flag.go +++ b/pkg/cfg/flag.go @@ -8,8 +8,14 @@ import ( "github.com/pkg/errors" ) -// Defaults registers flags using dst as the flagext.Registerer +// Defaults registers flags to the command line using dst as the +// flagext.Registerer func Defaults() Source { + return dDefaults(flag.CommandLine) +} + +// dDefaults registers flags to the flagSet using dst as the flagext.Registerer +func dDefaults(fs *flag.FlagSet) Source { return func(dst interface{}) error { r, ok := dst.(flagext.Registerer) if !ok { @@ -17,7 +23,7 @@ func Defaults() Source { } // already sets the defaults on r - r.RegisterFlags(flag.CommandLine) + r.RegisterFlags(fs) return nil } } @@ -25,13 +31,13 @@ func Defaults() Source { // Flags parses the flag from the command line, setting only user-supplied // values on the flagext.Registerer passed to Defaults() func Flags() Source { - return dFlags(os.Args[1:]) + return dFlags(flag.CommandLine, os.Args[1:]) } // dFlags parses the flagset, applying all values set on the slice -func dFlags(args []string) Source { +func dFlags(fs *flag.FlagSet, args []string) Source { return func(dst interface{}) error { // parse the final flagset - return flag.CommandLine.Parse(args) + return fs.Parse(args) } } diff --git a/pkg/cfg/flag_test.go b/pkg/cfg/flag_test.go index 7453cd6db8a59..967576a81b956 100644 --- a/pkg/cfg/flag_test.go +++ b/pkg/cfg/flag_test.go @@ -9,10 +9,16 @@ import ( "github.com/stretchr/testify/require" ) -// TestDefaults checks whether `FlagDefaults()` correctly sets values from flag defaults +// TestDefaults checks that defaults are correctly obtained from a +// flagext.Registerer func TestDefaults(t *testing.T) { - var d Data - err := Defaults(&flag.FlagSet{})(&d) + data := Data{} + fs := flag.NewFlagSet("testDefaults", flag.PanicOnError) + + err := Unmarshal(&data, + dDefaults(fs), + ) + require.NoError(t, err) assert.Equal(t, Data{ Verbose: false, @@ -21,8 +27,32 @@ func TestDefaults(t *testing.T) { Timeout: 60 * time.Second, }, TLS: TLS{ - Cert: "CERT", - Key: "KEY", + Cert: "DEFAULTCERT", + Key: "DEFAULTKEY", + }, + }, data) +} + +// TestFlags checks that defaults and flag values (they can't be separated) are +// correctly obtained from the command line +func TestFlags(t *testing.T) { + data := Data{} + fs := flag.NewFlagSet("testDefaults", flag.PanicOnError) + err := Unmarshal(&data, + dDefaults(fs), + dFlags(fs, []string{"-server.timeout=10h", "-verbose"}), + ) + require.NoError(t, err) + + assert.Equal(t, Data{ + Verbose: true, + Server: Server{ + Port: 80, + Timeout: 10 * time.Hour, + }, + TLS: TLS{ + Cert: "DEFAULTCERT", + Key: "DEFAULTKEY", }, - }, d) + }, data) } From 376de3e17285d8c02cff5819593f568e21b25f6c Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 26 Sep 2019 12:58:08 +0200 Subject: [PATCH 15/18] test(cfg): precedence tests Adds tests to ensure precendence is working correctly --- pkg/cfg/flag_test.go | 4 +-- pkg/cfg/precedence_test.go | 70 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 pkg/cfg/precedence_test.go diff --git a/pkg/cfg/flag_test.go b/pkg/cfg/flag_test.go index 967576a81b956..8c45a7846eff2 100644 --- a/pkg/cfg/flag_test.go +++ b/pkg/cfg/flag_test.go @@ -13,7 +13,7 @@ import ( // flagext.Registerer func TestDefaults(t *testing.T) { data := Data{} - fs := flag.NewFlagSet("testDefaults", flag.PanicOnError) + fs := flag.NewFlagSet(t.Name(), flag.PanicOnError) err := Unmarshal(&data, dDefaults(fs), @@ -37,7 +37,7 @@ func TestDefaults(t *testing.T) { // correctly obtained from the command line func TestFlags(t *testing.T) { data := Data{} - fs := flag.NewFlagSet("testDefaults", flag.PanicOnError) + fs := flag.NewFlagSet(t.Name(), flag.PanicOnError) err := Unmarshal(&data, dDefaults(fs), dFlags(fs, []string{"-server.timeout=10h", "-verbose"}), diff --git a/pkg/cfg/precedence_test.go b/pkg/cfg/precedence_test.go new file mode 100644 index 0000000000000..7232b6ff7b3a5 --- /dev/null +++ b/pkg/cfg/precedence_test.go @@ -0,0 +1,70 @@ +package cfg + +import ( + "flag" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// This file checks precedence rules are correctly working +// The default precedence recommended by this package is the following: +// flag defaults < yaml < user-set flags +// +// The following tests make sure that this is indeed correct + +const y = ` +verbose: true +tls: + cert: YAML +server: + port: 1234 +` + +func TestYAMLOverDefaults(t *testing.T) { + data := Data{} + fs := flag.NewFlagSet(t.Name(), flag.PanicOnError) + err := Unmarshal(&data, + dDefaults(fs), + dYAML([]byte(y)), + ) + + require.NoError(t, err) + assert.Equal(t, Data{ + Verbose: true, // yaml + Server: Server{ + Port: 1234, // yaml + Timeout: 60 * time.Second, // default + }, + TLS: TLS{ + Cert: "YAML", // yaml + Key: "DEFAULTKEY", // default + }, + }, data) +} + +func TestFlagOverYAML(t *testing.T) { + data := Data{} + fs := flag.NewFlagSet(t.Name(), flag.PanicOnError) + + err := Unmarshal(&data, + dDefaults(fs), + dYAML([]byte(y)), + dFlags(fs, []string{"-verbose=false", "-tls.cert=CLI"}), + ) + + require.NoError(t, err) + assert.Equal(t, Data{ + Verbose: false, // flag + Server: Server{ + Port: 1234, // yaml + Timeout: 60 * time.Second, // default + }, + TLS: TLS{ + Cert: "CLI", // flag + Key: "DEFAULTKEY", // default + }, + }, data) +} From 16774ed25eb89dca0477554ccb6f241a7da04bd1 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 26 Sep 2019 13:01:17 +0200 Subject: [PATCH 16/18] refactor(cfg): tidy TestParse() --- pkg/cfg/cfg_test.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/pkg/cfg/cfg_test.go b/pkg/cfg/cfg_test.go index 87192f0de0290..b0de7e4309aa1 100644 --- a/pkg/cfg/cfg_test.go +++ b/pkg/cfg/cfg_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -16,22 +17,27 @@ server: tls: key: YAML `)) - fs := flag.NewFlagSet("testParse", flag.PanicOnError) + + fs := flag.NewFlagSet(t.Name(), flag.PanicOnError) flagSource := dFlags(fs, []string{"-verbose", "-server.port=21"}) - var c Data - err := dParse(&c, dDefaults(fs), yamlSource, flagSource) + data := Data{} + err := dParse(&data, + dDefaults(fs), + yamlSource, + flagSource, + ) require.NoError(t, err) - require.Equal(t, Data{ - Verbose: true, + assert.Equal(t, Data{ + Verbose: true, // flag Server: Server{ - Port: 21, - Timeout: 60 * time.Hour, + Port: 21, // flag + Timeout: 60 * time.Hour, // defaults }, TLS: TLS{ - Cert: "DEFAULTCERT", - Key: "YAML", + Cert: "DEFAULTCERT", // defaults + Key: "YAML", // yaml }, - }, c) + }, data) } From 0082dcbadebe6bb092d48504f78cd1cefb988700 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 26 Sep 2019 15:45:10 +0200 Subject: [PATCH 17/18] feat(cfg): ignore empty json config --- pkg/cfg/files.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/cfg/files.go b/pkg/cfg/files.go index c3f7308350a26..7cedbc81fd529 100644 --- a/pkg/cfg/files.go +++ b/pkg/cfg/files.go @@ -11,6 +11,9 @@ import ( // JSON returns a Source that opens the supplied `.json` file and loads it. func JSON(f *string) Source { return func(dst interface{}) error { + if f == nil { + return nil + } j, err := ioutil.ReadFile(*f) if err != nil { From 4b11ee8aab9870f0b41c4cb54909e22c283295a4 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 26 Sep 2019 15:58:40 +0200 Subject: [PATCH 18/18] fix: use correct logger --- Gopkg.lock | 2 ++ cmd/loki/main.go | 7 ++++--- cmd/promtail/main.go | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index c029adf7ee985..5b4b2775e3877 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1825,6 +1825,8 @@ "github.com/cortexproject/cortex/pkg/ingester/client", "github.com/cortexproject/cortex/pkg/ingester/index", "github.com/cortexproject/cortex/pkg/ring", + "github.com/cortexproject/cortex/pkg/ring/kv", + "github.com/cortexproject/cortex/pkg/ring/kv/codec", "github.com/cortexproject/cortex/pkg/util", "github.com/cortexproject/cortex/pkg/util/flagext", "github.com/cortexproject/cortex/pkg/util/grpcclient", diff --git a/cmd/loki/main.go b/cmd/loki/main.go index a8b4a07bbebff..abea6d2751af5 100644 --- a/cmd/loki/main.go +++ b/cmd/loki/main.go @@ -3,7 +3,6 @@ package main import ( "flag" "fmt" - "log" "os" "reflect" @@ -28,7 +27,8 @@ func main() { var config loki.Config if err := cfg.Parse(&config); err != nil { - log.Fatalln(err) + level.Error(util.Logger).Log("msg", "parsing config", "error", err) + os.Exit(1) } if *printVersion { fmt.Print(version.Print("loki")) @@ -39,7 +39,8 @@ func main() { // call it atleast once, the defaults are set to an empty struct. // We call it with the flag values so that the config file unmarshalling only overrides the values set in the config. if _, err := validation.NewOverrides(config.LimitsConfig); err != nil { - log.Fatalln(err) + level.Error(util.Logger).Log("msg", "setting up overrides", "error", err) + os.Exit(1) } // Init the logger which will honor the log level set in config.Server diff --git a/cmd/promtail/main.go b/cmd/promtail/main.go index 51ef9b4b2cf6d..bda02a679dea4 100644 --- a/cmd/promtail/main.go +++ b/cmd/promtail/main.go @@ -3,7 +3,6 @@ package main import ( "flag" "fmt" - "log" "os" "reflect" @@ -28,7 +27,8 @@ func main() { var config config.Config if err := cfg.Parse(&config); err != nil { - log.Fatalln(err) + level.Error(util.Logger).Log("msg", "parsing config", "error", err) + os.Exit(1) } if *printVersion { fmt.Print(version.Print("promtail"))