From d5b5ab6aa389a2530f5fbd3c03798cda61419d98 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 6 Feb 2020 16:17:44 -0500 Subject: [PATCH 1/9] --dry-run Promtail. Signed-off-by: Cyril Tovena --- cmd/promtail/main.go | 3 +- go.mod | 1 + go.sum | 2 ++ pkg/promtail/client/logger.go | 56 ++++++++++++++++++++++++++++++ pkg/promtail/client/logger_test.go | 21 +++++++++++ pkg/promtail/promtail.go | 20 +++++++---- pkg/promtail/promtail_test.go | 26 +++++++++++++- 7 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 pkg/promtail/client/logger.go create mode 100644 pkg/promtail/client/logger_test.go diff --git a/cmd/promtail/main.go b/cmd/promtail/main.go index 36264575f2e15..2fffe3c0ad9d9 100644 --- a/cmd/promtail/main.go +++ b/cmd/promtail/main.go @@ -27,6 +27,7 @@ func init() { func main() { printVersion := flag.Bool("version", false, "Print this builds version information") + dryRun := flag.Bool("dry-run", false, "Start Promtail but print entries instead of sending them to Loki.") // Load config, merging config file and CLI flags var config config.Config @@ -57,7 +58,7 @@ func main() { stages.Debug = true } - p, err := promtail.New(config) + p, err := promtail.New(config, *dryRun) if err != nil { level.Error(util.Logger).Log("msg", "error creating promtail", "error", err) os.Exit(1) diff --git a/go.mod b/go.mod index df9ab2d6c86e5..a865b1695a259 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/golang/snappy v0.0.1 github.com/gorilla/mux v1.7.1 github.com/gorilla/websocket v1.4.0 + github.com/grafana/tail v0.0.0-20200127140945-4647d4b312f2 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 github.com/hashicorp/golang-lru v0.5.3 diff --git a/go.sum b/go.sum index 69cb52152aec2..fa5018d7d1013 100644 --- a/go.sum +++ b/go.sum @@ -395,6 +395,8 @@ github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grafana/tail v0.0.0-20191024143944-0b54ddf21fe7 h1:eeBhshivxpgHEX78QxJkoL251Pjr0B2GL59ZsivnplU= github.com/grafana/tail v0.0.0-20191024143944-0b54ddf21fe7/go.mod h1:aS6CMYGLEIABOzX3OL8SqZ3zAZCGN7nmBnqgnyJGxyA= +github.com/grafana/tail v0.0.0-20200127140945-4647d4b312f2 h1:rleaNROLYWaF/fN7hgk/j+YzD7hPKfJjtwVf3JtgKrA= +github.com/grafana/tail v0.0.0-20200127140945-4647d4b312f2/go.mod h1:aS6CMYGLEIABOzX3OL8SqZ3zAZCGN7nmBnqgnyJGxyA= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= diff --git a/pkg/promtail/client/logger.go b/pkg/promtail/client/logger.go new file mode 100644 index 0000000000000..600435c803a9d --- /dev/null +++ b/pkg/promtail/client/logger.go @@ -0,0 +1,56 @@ +package client + +import ( + "fmt" + "os" + "sync" + "text/tabwriter" + "time" + + "github.com/cortexproject/cortex/pkg/util" + "github.com/fatih/color" + "github.com/prometheus/common/model" + "gopkg.in/yaml.v2" +) + +type logger struct { + *tabwriter.Writer + sync.Mutex +} + +// NewLogger creates a new client logger that logs entries instead of sending them. +func NewLogger(cfgs ...Config) (Client, error) { + // make sure the clients config is valid + c, err := NewMulti(util.Logger, cfgs...) + if err != nil { + return nil, err + } + c.Stop() + fmt.Println(color.YellowString("Clients configured:")) + for _, cfg := range cfgs { + yaml, err := yaml.Marshal(cfg) + if err != nil { + return nil, err + } + fmt.Println("----------------------") + fmt.Println(string(yaml)) + } + return &logger{ + Writer: tabwriter.NewWriter(os.Stdout, 0, 4, 0, '\t', 0), + }, nil +} + +func (*logger) Stop() {} + +func (l *logger) Handle(labels model.LabelSet, time time.Time, entry string) error { + l.Lock() + defer l.Unlock() + fmt.Fprint(l.Writer, color.BlueString(time.Format("2006-01-02T15:04:05"))) + fmt.Fprint(l.Writer, "\t") + fmt.Fprint(l.Writer, color.YellowString(labels.String())) + fmt.Fprint(l.Writer, "\t") + fmt.Fprint(l.Writer, entry) + fmt.Fprint(l.Writer, "\n") + l.Flush() + return nil +} diff --git a/pkg/promtail/client/logger_test.go b/pkg/promtail/client/logger_test.go new file mode 100644 index 0000000000000..3ff0e948e66aa --- /dev/null +++ b/pkg/promtail/client/logger_test.go @@ -0,0 +1,21 @@ +package client + +import ( + "net/url" + "testing" + "time" + + "github.com/cortexproject/cortex/pkg/util/flagext" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" +) + +func TestNewLogger(t *testing.T) { + _, err := NewLogger([]Config{}...) + require.Error(t, err) + + l, err := NewLogger([]Config{{URL: flagext.URLValue{URL: &url.URL{Host: "string"}}}}...) + require.NoError(t, err) + err = l.Handle(model.LabelSet{"foo": "bar"}, time.Now(), "entry") + require.NoError(t, err) +} diff --git a/pkg/promtail/promtail.go b/pkg/promtail/promtail.go index 3fae7132c7083..f256a6233c8c7 100644 --- a/pkg/promtail/promtail.go +++ b/pkg/promtail/promtail.go @@ -19,7 +19,7 @@ type Promtail struct { } // New makes a new Promtail. -func New(cfg config.Config) (*Promtail, error) { +func New(cfg config.Config, dryRun bool) (*Promtail, error) { positions, err := positions.New(util.Logger, cfg.PositionsConfig) if err != nil { return nil, err @@ -30,12 +30,20 @@ func New(cfg config.Config) (*Promtail, error) { cfg.ClientConfigs = append(cfg.ClientConfigs, cfg.ClientConfig) } - client, err := client.NewMulti(util.Logger, cfg.ClientConfigs...) - if err != nil { - return nil, err + var cl client.Client + if dryRun { + cl, err = client.NewLogger(cfg.ClientConfigs...) + if err != nil { + return nil, err + } + } else { + cl, err = client.NewMulti(util.Logger, cfg.ClientConfigs...) + if err != nil { + return nil, err + } } - tms, err := targets.NewTargetManagers(util.Logger, positions, client, cfg.ScrapeConfig, &cfg.TargetConfig) + tms, err := targets.NewTargetManagers(util.Logger, positions, cl, cfg.ScrapeConfig, &cfg.TargetConfig) if err != nil { return nil, err } @@ -46,7 +54,7 @@ func New(cfg config.Config) (*Promtail, error) { } return &Promtail{ - client: client, + client: cl, positions: positions, targetManagers: tms, server: server, diff --git a/pkg/promtail/promtail_test.go b/pkg/promtail/promtail_test.go index c117af245a171..8e8a84f424cd8 100644 --- a/pkg/promtail/promtail_test.go +++ b/pkg/promtail/promtail_test.go @@ -7,6 +7,7 @@ import ( "math" "math/rand" "net/http" + "net/url" "os" "path/filepath" "reflect" @@ -26,11 +27,14 @@ import ( "github.com/prometheus/prometheus/pkg/textparse" "github.com/prometheus/prometheus/promql" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/logentry/stages" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/promtail/api" + "github.com/grafana/loki/pkg/promtail/client" "github.com/grafana/loki/pkg/promtail/config" + "github.com/grafana/loki/pkg/promtail/positions" "github.com/grafana/loki/pkg/promtail/scrape" "github.com/grafana/loki/pkg/promtail/targets" ) @@ -84,7 +88,7 @@ func TestPromtail(t *testing.T) { // Run. - p, err := New(buildTestConfig(t, positionsFileName, testDir)) + p, err := New(buildTestConfig(t, positionsFileName, testDir), false) if err != nil { t.Error("error creating promtail", err) return @@ -628,3 +632,23 @@ func randName() string { } return string(b) } + +func Test_DryRun(t *testing.T) { + + f, err := ioutil.TempFile("/tmp", "Test_DryRun") + require.NoError(t, err) + defer os.Remove(f.Name()) + + _, err = New(config.Config{}, true) + require.Error(t, err) + + p, err := New(config.Config{ + ClientConfig: client.Config{URL: flagext.URLValue{URL: &url.URL{Host: "string"}}}, + PositionsConfig: positions.Config{ + PositionsFile: f.Name(), + SyncPeriod: time.Second, + }, + }, false) + require.NoError(t, err) + require.IsType(t, client.MultiClient{}, p.client) +} From eea212f7b9008555def5ef35135aa22abec5b141 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 6 Feb 2020 16:20:33 -0500 Subject: [PATCH 2/9] Remove changed on gomod Signed-off-by: Cyril Tovena --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index a865b1695a259..df9ab2d6c86e5 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,6 @@ require ( github.com/golang/snappy v0.0.1 github.com/gorilla/mux v1.7.1 github.com/gorilla/websocket v1.4.0 - github.com/grafana/tail v0.0.0-20200127140945-4647d4b312f2 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 github.com/hashicorp/golang-lru v0.5.3 diff --git a/go.sum b/go.sum index fa5018d7d1013..69cb52152aec2 100644 --- a/go.sum +++ b/go.sum @@ -395,8 +395,6 @@ github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grafana/tail v0.0.0-20191024143944-0b54ddf21fe7 h1:eeBhshivxpgHEX78QxJkoL251Pjr0B2GL59ZsivnplU= github.com/grafana/tail v0.0.0-20191024143944-0b54ddf21fe7/go.mod h1:aS6CMYGLEIABOzX3OL8SqZ3zAZCGN7nmBnqgnyJGxyA= -github.com/grafana/tail v0.0.0-20200127140945-4647d4b312f2 h1:rleaNROLYWaF/fN7hgk/j+YzD7hPKfJjtwVf3JtgKrA= -github.com/grafana/tail v0.0.0-20200127140945-4647d4b312f2/go.mod h1:aS6CMYGLEIABOzX3OL8SqZ3zAZCGN7nmBnqgnyJGxyA= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= From fc8248bb5de30eefe308d71f7f1ae47eab4023b8 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 6 Feb 2020 17:07:38 -0500 Subject: [PATCH 3/9] Fixes flaky test Signed-off-by: Cyril Tovena --- pkg/promtail/promtail_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/promtail/promtail_test.go b/pkg/promtail/promtail_test.go index 8e8a84f424cd8..c34477cc7e202 100644 --- a/pkg/promtail/promtail_test.go +++ b/pkg/promtail/promtail_test.go @@ -20,6 +20,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" sd_config "github.com/prometheus/prometheus/discovery/config" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -642,6 +643,8 @@ func Test_DryRun(t *testing.T) { _, err = New(config.Config{}, true) require.Error(t, err) + prometheus.DefaultRegisterer = prometheus.NewRegistry() // reset registry, otherwise you can't create 2 weavework server. + p, err := New(config.Config{ ClientConfig: client.Config{URL: flagext.URLValue{URL: &url.URL{Host: "string"}}}, PositionsConfig: positions.Config{ From 783efecd67d2a3602f62079246fb370890687750 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Tue, 11 Feb 2020 13:50:42 -0500 Subject: [PATCH 4/9] Ensure position file is readonly when doing dry-run Signed-off-by: Cyril Tovena --- pkg/promtail/positions/positions.go | 72 ++++++++++++-------- pkg/promtail/positions/positions_test.go | 44 ++++++++++++ pkg/promtail/promtail.go | 1 + pkg/promtail/promtail_test.go | 10 +++ pkg/promtail/targets/filetarget.go | 4 +- pkg/promtail/targets/filetargetmanager.go | 4 +- pkg/promtail/targets/journaltargetmanager.go | 2 +- pkg/promtail/targets/manager.go | 2 +- pkg/promtail/targets/tailer.go | 4 +- 9 files changed, 106 insertions(+), 37 deletions(-) diff --git a/pkg/promtail/positions/positions.go b/pkg/promtail/positions/positions.go index 05d777ee23f1e..5f8eac2d59331 100644 --- a/pkg/promtail/positions/positions.go +++ b/pkg/promtail/positions/positions.go @@ -23,6 +23,7 @@ type Config struct { SyncPeriod time.Duration `yaml:"sync_period"` PositionsFile string `yaml:"filename"` IgnoreInvalidYaml bool `yaml:"ignore_invalid_yaml"` + ReadOnly bool `yaml:"-"` } // RegisterFlags register flags. @@ -33,7 +34,7 @@ func (cfg *Config) RegisterFlags(flags *flag.FlagSet) { } // Positions tracks how far through each file we've read. -type Positions struct { +type positions struct { logger log.Logger cfg Config mtx sync.Mutex @@ -47,17 +48,40 @@ type File struct { Positions map[string]string `yaml:"positions"` } +type Positions interface { + // GetString returns how far we've through a file as a string. + // JournalTarget writes a journal cursor to the positions file, while + // FileTarget writes an integer offset. Use Get to read the integer + // offset. + GetString(path string) string + // Get returns how far we've read through a file. Returns an error + // if the value stored for the file is not an integer. + Get(path string) (int64, error) + // PutString records (asynchronsouly) how far we've read through a file. + // Unlike Put, it records a string offset and is only useful for + // JournalTargets which doesn't have integer offsets. + PutString(path string, pos string) + // Put records (asynchronously) how far we've read through a file. + Put(path string, pos int64) + // Remove removes the position tracking for a filepath + Remove(path string) + // SyncPeriod returns how often the positions file gets resynced + SyncPeriod() time.Duration + // Stop the Position tracker. + Stop() +} + // New makes a new Positions. -func New(logger log.Logger, cfg Config) (*Positions, error) { - positions, err := readPositionsFile(cfg, logger) +func New(logger log.Logger, cfg Config) (*positions, error) { + positionData, err := readPositionsFile(cfg, logger) if err != nil { return nil, err } - p := &Positions{ + p := &positions{ logger: logger, cfg: cfg, - positions: positions, + positions: positionData, quit: make(chan struct{}), done: make(chan struct{}), } @@ -66,39 +90,28 @@ func New(logger log.Logger, cfg Config) (*Positions, error) { return p, nil } -// Stop the Position tracker. -func (p *Positions) Stop() { +func (p *positions) Stop() { close(p.quit) <-p.done } -// PutString records (asynchronsouly) how far we've read through a file. -// Unlike Put, it records a string offset and is only useful for -// JournalTargets which doesn't have integer offsets. -func (p *Positions) PutString(path string, pos string) { +func (p *positions) PutString(path string, pos string) { p.mtx.Lock() defer p.mtx.Unlock() p.positions[path] = pos } -// Put records (asynchronously) how far we've read through a file. -func (p *Positions) Put(path string, pos int64) { +func (p *positions) Put(path string, pos int64) { p.PutString(path, strconv.FormatInt(pos, 10)) } -// GetString returns how far we've through a file as a string. -// JournalTarget writes a journal cursor to the positions file, while -// FileTarget writes an integer offset. Use Get to read the integer -// offset. -func (p *Positions) GetString(path string) string { +func (p *positions) GetString(path string) string { p.mtx.Lock() defer p.mtx.Unlock() return p.positions[path] } -// Get returns how far we've read through a file. Returns an error -// if the value stored for the file is not an integer. -func (p *Positions) Get(path string) (int64, error) { +func (p *positions) Get(path string) (int64, error) { p.mtx.Lock() defer p.mtx.Unlock() pos, ok := p.positions[path] @@ -108,23 +121,21 @@ func (p *Positions) Get(path string) (int64, error) { return strconv.ParseInt(pos, 10, 64) } -// Remove removes the position tracking for a filepath -func (p *Positions) Remove(path string) { +func (p *positions) Remove(path string) { p.mtx.Lock() defer p.mtx.Unlock() p.remove(path) } -func (p *Positions) remove(path string) { +func (p *positions) remove(path string) { delete(p.positions, path) } -// SyncPeriod returns how often the positions file gets resynced -func (p *Positions) SyncPeriod() time.Duration { +func (p *positions) SyncPeriod() time.Duration { return p.cfg.SyncPeriod } -func (p *Positions) run() { +func (p *positions) run() { defer func() { p.save() level.Debug(p.logger).Log("msg", "positions saved") @@ -143,7 +154,10 @@ func (p *Positions) run() { } } -func (p *Positions) save() { +func (p *positions) save() { + if p.cfg.ReadOnly { + return + } p.mtx.Lock() positions := make(map[string]string, len(p.positions)) for k, v := range p.positions { @@ -156,7 +170,7 @@ func (p *Positions) save() { } } -func (p *Positions) cleanup() { +func (p *positions) cleanup() { p.mtx.Lock() defer p.mtx.Unlock() toRemove := []string{} diff --git a/pkg/promtail/positions/positions_test.go b/pkg/promtail/positions/positions_test.go index c391aa2348af2..294ba84258380 100644 --- a/pkg/promtail/positions/positions_test.go +++ b/pkg/promtail/positions/positions_test.go @@ -5,7 +5,9 @@ import ( "os" "strings" "testing" + "time" + "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" ) @@ -136,3 +138,45 @@ func TestReadPositionsFromBadYamlIgnoreCorruption(t *testing.T) { require.NoError(t, err) require.Equal(t, map[string]string{}, out) } + +func Test_ReadOnly(t *testing.T) { + temp := tempFilename(t) + defer func() { + _ = os.Remove(temp) + }() + yaml := []byte(`positions: + /tmp/random.log: "17623" +`) + err := ioutil.WriteFile(temp, yaml, 0644) + if err != nil { + t.Fatal(err) + } + p, err := New(util.Logger, Config{ + SyncPeriod: 20 * time.Nanosecond, + PositionsFile: temp, + ReadOnly: true, + }) + if err != nil { + t.Fatal(err) + } + defer p.Stop() + p.Put("/foo/bar/f", 12132132) + p.PutString("/foo/f", "100") + pos, err := p.Get("/tmp/random.log") + if err != nil { + t.Fatal(err) + } + require.Equal(t, int64(17623), pos) + p.save() + out, err := readPositionsFile(Config{ + PositionsFile: temp, + IgnoreInvalidYaml: true, + ReadOnly: true, + }, log.NewNopLogger()) + + require.NoError(t, err) + require.Equal(t, map[string]string{ + "/tmp/random.log": "17623", + }, out) + +} diff --git a/pkg/promtail/promtail.go b/pkg/promtail/promtail.go index e5767dec825a3..50dd4ccefe150 100644 --- a/pkg/promtail/promtail.go +++ b/pkg/promtail/promtail.go @@ -36,6 +36,7 @@ func New(cfg config.Config, dryRun bool) (*Promtail, error) { if err != nil { return nil, err } + cfg.PositionsConfig.ReadOnly = true } else { cl, err = client.NewMulti(util.Logger, cfg.ClientConfigs...) if err != nil { diff --git a/pkg/promtail/promtail_test.go b/pkg/promtail/promtail_test.go index c34477cc7e202..f4c76548adb3a 100644 --- a/pkg/promtail/promtail_test.go +++ b/pkg/promtail/promtail_test.go @@ -644,6 +644,16 @@ func Test_DryRun(t *testing.T) { require.Error(t, err) prometheus.DefaultRegisterer = prometheus.NewRegistry() // reset registry, otherwise you can't create 2 weavework server. + _, err = New(config.Config{ + ClientConfig: client.Config{URL: flagext.URLValue{URL: &url.URL{Host: "string"}}}, + PositionsConfig: positions.Config{ + PositionsFile: f.Name(), + SyncPeriod: time.Second, + }, + }, true) + require.NoError(t, err) + + prometheus.DefaultRegisterer = prometheus.NewRegistry() p, err := New(config.Config{ ClientConfig: client.Config{URL: flagext.URLValue{URL: &url.URL{Host: "string"}}}, diff --git a/pkg/promtail/targets/filetarget.go b/pkg/promtail/targets/filetarget.go index 2a70dd1e36161..4bb02b494c960 100644 --- a/pkg/promtail/targets/filetarget.go +++ b/pkg/promtail/targets/filetarget.go @@ -68,7 +68,7 @@ type FileTarget struct { logger log.Logger handler api.EntryHandler - positions *positions.Positions + positions positions.Positions labels model.LabelSet discoveredLabels model.LabelSet @@ -84,7 +84,7 @@ type FileTarget struct { } // NewFileTarget create a new FileTarget. -func NewFileTarget(logger log.Logger, handler api.EntryHandler, positions *positions.Positions, path string, labels model.LabelSet, discoveredLabels model.LabelSet, targetConfig *Config) (*FileTarget, error) { +func NewFileTarget(logger log.Logger, handler api.EntryHandler, positions positions.Positions, path string, labels model.LabelSet, discoveredLabels model.LabelSet, targetConfig *Config) (*FileTarget, error) { watcher, err := fsnotify.NewWatcher() if err != nil { diff --git a/pkg/promtail/targets/filetargetmanager.go b/pkg/promtail/targets/filetargetmanager.go index a096c7192c948..4ce42593913b4 100644 --- a/pkg/promtail/targets/filetargetmanager.go +++ b/pkg/promtail/targets/filetargetmanager.go @@ -54,7 +54,7 @@ type FileTargetManager struct { // NewFileTargetManager creates a new TargetManager. func NewFileTargetManager( logger log.Logger, - positions *positions.Positions, + positions positions.Positions, client api.EntryHandler, scrapeConfigs []scrape.Config, targetConfig *Config, @@ -178,7 +178,7 @@ func (tm *FileTargetManager) AllTargets() map[string][]Target { // targetSyncer sync targets based on service discovery changes. type targetSyncer struct { log log.Logger - positions *positions.Positions + positions positions.Positions entryHandler api.EntryHandler hostname string diff --git a/pkg/promtail/targets/journaltargetmanager.go b/pkg/promtail/targets/journaltargetmanager.go index 46034e7de2f93..b6f3641ce8cf3 100644 --- a/pkg/promtail/targets/journaltargetmanager.go +++ b/pkg/promtail/targets/journaltargetmanager.go @@ -17,7 +17,7 @@ type JournalTargetManager struct{} // on this platform. func NewJournalTargetManager( logger log.Logger, - positions *positions.Positions, + positions positions.Positions, client api.EntryHandler, scrapeConfigs []scrape.Config, ) (*JournalTargetManager, error) { diff --git a/pkg/promtail/targets/manager.go b/pkg/promtail/targets/manager.go index 87020a2f70908..eaf58c9ab0767 100644 --- a/pkg/promtail/targets/manager.go +++ b/pkg/promtail/targets/manager.go @@ -20,7 +20,7 @@ type targetManager interface { // TargetManagers manages a list of target managers. type TargetManagers struct { targetManagers []targetManager - positions *positions.Positions + positions positions.Positions } // NewTargetManagers makes a new TargetManagers diff --git a/pkg/promtail/targets/tailer.go b/pkg/promtail/targets/tailer.go index ed44867406d2d..c15ce32b98d88 100644 --- a/pkg/promtail/targets/tailer.go +++ b/pkg/promtail/targets/tailer.go @@ -18,7 +18,7 @@ import ( type tailer struct { logger log.Logger handler api.EntryHandler - positions *positions.Positions + positions positions.Positions path string tail *tail.Tail @@ -29,7 +29,7 @@ type tailer struct { done chan struct{} } -func newTailer(logger log.Logger, handler api.EntryHandler, positions *positions.Positions, path string) (*tailer, error) { +func newTailer(logger log.Logger, handler api.EntryHandler, positions positions.Positions, path string) (*tailer, error) { // Simple check to make sure the file we are tailing doesn't // have a position already saved which is past the end of the file. fi, err := os.Stat(path) From d09bc1ee023caec4f1d354c23bbdc5d43b44a5c9 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Tue, 11 Feb 2020 16:09:00 -0500 Subject: [PATCH 5/9] Add information about dry run mode in the troubleshooting section. Signed-off-by: Cyril Tovena --- docs/clients/promtail/troubleshooting.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/clients/promtail/troubleshooting.md b/docs/clients/promtail/troubleshooting.md index 58c98e2272698..ab0fbc79848b8 100644 --- a/docs/clients/promtail/troubleshooting.md +++ b/docs/clients/promtail/troubleshooting.md @@ -3,6 +3,19 @@ This document describes known failure modes of `promtail` on edge cases and the adopted trade-offs. +## Dry running + +Promtail can be configured to print log stream entries instead of sending them to Loki. +This can be used in combination with [piping data](#pipe-data-to-promtail) to debug or troubleshoot promtail log parsing. + +In dry run mode, Promtail still support reading from a [positions](configuration.md#position_config) file however no update will be made to the targeted file, this is to ensure you can easily retry the same set of lines. + +To start Promtail in dry run mode use the flag `--dry-run` as shown in the example below: + +```bash +cat my.log | promtail --dry-run --client.url http://127.0.0.1:3100/loki/api/v1/push +``` + ## Pipe data to Promtail Promtail supports piping data for sending logs to Loki. This is a very useful way to troubleshooting your configuration. From 9f1e4cddbe09e4780bc8240d7dfe1af374dce9e5 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Tue, 11 Feb 2020 16:34:20 -0500 Subject: [PATCH 6/9] Add more spaces. Signed-off-by: Cyril Tovena --- pkg/promtail/client/logger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/promtail/client/logger.go b/pkg/promtail/client/logger.go index 600435c803a9d..5b8f3c412bf0d 100644 --- a/pkg/promtail/client/logger.go +++ b/pkg/promtail/client/logger.go @@ -36,7 +36,7 @@ func NewLogger(cfgs ...Config) (Client, error) { fmt.Println(string(yaml)) } return &logger{ - Writer: tabwriter.NewWriter(os.Stdout, 0, 4, 0, '\t', 0), + Writer: tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0), }, nil } From 7c9b0ad7a1f1d435bda566a2aaac564471ab271c Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Tue, 11 Feb 2020 16:37:46 -0500 Subject: [PATCH 7/9] Fixes journaltarget refactor. Signed-off-by: Cyril Tovena --- pkg/promtail/targets/journaltarget.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/promtail/targets/journaltarget.go b/pkg/promtail/targets/journaltarget.go index 10c6a5cba3184..905ff94bc6430 100644 --- a/pkg/promtail/targets/journaltarget.go +++ b/pkg/promtail/targets/journaltarget.go @@ -78,7 +78,7 @@ var defaultJournalEntryFunc = func(c sdjournal.JournalReaderConfig, cursor strin type JournalTarget struct { logger log.Logger handler api.EntryHandler - positions *positions.Positions + positions positions.Positions positionPath string relabelConfig []*relabel.Config config *scrape.JournalTargetConfig @@ -92,7 +92,7 @@ type JournalTarget struct { func NewJournalTarget( logger log.Logger, handler api.EntryHandler, - positions *positions.Positions, + positions positions.Positions, jobName string, relabelConfig []*relabel.Config, targetConfig *scrape.JournalTargetConfig, @@ -113,7 +113,7 @@ func NewJournalTarget( func journalTargetWithReader( logger log.Logger, handler api.EntryHandler, - positions *positions.Positions, + positions positions.Positions, jobName string, relabelConfig []*relabel.Config, targetConfig *scrape.JournalTargetConfig, From b2e59759cd3a5d3f653b8c3247902c02e0b576b3 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Tue, 11 Feb 2020 16:46:11 -0500 Subject: [PATCH 8/9] Fixes journaltarget refactor. Signed-off-by: Cyril Tovena --- pkg/promtail/targets/journaltargetmanager_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/promtail/targets/journaltargetmanager_linux.go b/pkg/promtail/targets/journaltargetmanager_linux.go index d538a3e7ac0ca..f4a8e45ee36bd 100644 --- a/pkg/promtail/targets/journaltargetmanager_linux.go +++ b/pkg/promtail/targets/journaltargetmanager_linux.go @@ -21,7 +21,7 @@ type JournalTargetManager struct { // NewJournalTargetManager creates a new JournalTargetManager. func NewJournalTargetManager( logger log.Logger, - positions *positions.Positions, + positions positions.Positions, client api.EntryHandler, scrapeConfigs []scrape.Config, ) (*JournalTargetManager, error) { From 35e4a8ef1616cdeff1c1d19fc883cd9f35dcb27d Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Tue, 11 Feb 2020 17:08:08 -0500 Subject: [PATCH 9/9] Fixes journaltarget refactor. Signed-off-by: Cyril Tovena --- pkg/promtail/positions/positions.go | 2 +- pkg/promtail/positions/positions_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/promtail/positions/positions.go b/pkg/promtail/positions/positions.go index 5f8eac2d59331..202ba65b533f3 100644 --- a/pkg/promtail/positions/positions.go +++ b/pkg/promtail/positions/positions.go @@ -72,7 +72,7 @@ type Positions interface { } // New makes a new Positions. -func New(logger log.Logger, cfg Config) (*positions, error) { +func New(logger log.Logger, cfg Config) (Positions, error) { positionData, err := readPositionsFile(cfg, logger) if err != nil { return nil, err diff --git a/pkg/promtail/positions/positions_test.go b/pkg/promtail/positions/positions_test.go index 294ba84258380..0919c62cf9122 100644 --- a/pkg/promtail/positions/positions_test.go +++ b/pkg/promtail/positions/positions_test.go @@ -167,7 +167,7 @@ func Test_ReadOnly(t *testing.T) { t.Fatal(err) } require.Equal(t, int64(17623), pos) - p.save() + p.(*positions).save() out, err := readPositionsFile(Config{ PositionsFile: temp, IgnoreInvalidYaml: true,