From c994823369d65785e72c4247fd50c656801e429a Mon Sep 17 00:00:00 2001
From: Edward Welch <edward.welch@grafana.com>
Date: Tue, 26 Mar 2019 16:04:46 -0400
Subject: [PATCH] Change hpcloud tailer library to poll mode to workaround bugs
 in the event driven code within the library Had to rework tests with some
 waits for messages because polling is slower There are several longer sleeps
 in the promtail_test which are not ideal and have been marked with FIXME to
 remove once we have a better solution

---
 pkg/promtail/promtail_test.go           | 13 ++++++++
 pkg/promtail/targets/filetarget_test.go | 42 +++++++++++++++++++++++++
 pkg/promtail/targets/tailer.go          |  1 +
 3 files changed, 56 insertions(+)

diff --git a/pkg/promtail/promtail_test.go b/pkg/promtail/promtail_test.go
index 304502940fb45..4db825dec56d0 100644
--- a/pkg/promtail/promtail_test.go
+++ b/pkg/promtail/promtail_test.go
@@ -179,6 +179,10 @@ func fileRoll(t *testing.T, filename string, prefix string) int {
 		}
 		time.Sleep(1 * time.Millisecond)
 	}
+
+	//FIXME this is a hack to make sure the hpcloud tail polling implementation reads all lines before we roll the file
+	time.Sleep(300 * time.Millisecond)
+
 	if err = os.Rename(filename, filename+".1"); err != nil {
 		t.Fatal("Failed to rename file for test: ", err)
 	}
@@ -195,6 +199,9 @@ func fileRoll(t *testing.T, filename string, prefix string) int {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	//FIXME this is a hack to make sure the hpcloud tail polling implementation reads all lines before we roll the file
+	time.Sleep(300 * time.Millisecond)
+
 	return 200
 }
 
@@ -224,6 +231,9 @@ func symlinkRoll(t *testing.T, testDir string, filename string, prefix string) i
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	//FIXME this is a hack to make sure the hpcloud tail polling implementation reads all lines before we roll the file
+	time.Sleep(300 * time.Millisecond)
+
 	// Remove the link, make a new file, link to the new file.
 	if err := os.Remove(filename); err != nil {
 		t.Fatal(err)
@@ -245,6 +255,9 @@ func symlinkRoll(t *testing.T, testDir string, filename string, prefix string) i
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	//FIXME this is a hack to make sure the hpcloud tail polling implementation reads all lines before we roll the file
+	time.Sleep(300 * time.Millisecond)
+
 	return 200
 
 }
diff --git a/pkg/promtail/targets/filetarget_test.go b/pkg/promtail/targets/filetarget_test.go
index bf4109fc09b47..e766013a3ae17 100644
--- a/pkg/promtail/targets/filetarget_test.go
+++ b/pkg/promtail/targets/filetarget_test.go
@@ -68,6 +68,12 @@ func TestLongPositionsSyncDelayStillSavesCorrectPosition(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown := 10000
+	for len(client.messages) != 10 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	target.Stop()
 	ps.Stop()
 
@@ -155,6 +161,12 @@ func TestWatchEntireDirectory(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown := 10000
+	for len(client.messages) != 10 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	target.Stop()
 	ps.Stop()
 
@@ -238,6 +250,12 @@ func TestFileRolls(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown := 10000
+	for len(client.messages) != 10 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	// Rename the log file to something not in the pattern, then create a new file with the same name.
 	err = os.Rename(logFile, dirName+"/test.log.1")
 	if err != nil {
@@ -256,6 +274,12 @@ func TestFileRolls(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown = 10000
+	for len(client.messages) != 20 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	target.Stop()
 	positions.Stop()
 
@@ -324,6 +348,12 @@ func TestResumesWhereLeftOff(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown := 10000
+	for len(client.messages) != 10 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	target.Stop()
 	ps.Stop()
 
@@ -352,6 +382,12 @@ func TestResumesWhereLeftOff(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown = 10000
+	for len(client.messages) != 20 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	target2.Stop()
 	ps2.Stop()
 
@@ -431,6 +467,12 @@ func TestGlobWithMultipleFiles(t *testing.T) {
 		time.Sleep(1 * time.Millisecond)
 	}
 
+	countdown := 10000
+	for len(client.messages) != 20 && countdown > 0 {
+		time.Sleep(1 * time.Millisecond)
+		countdown--
+	}
+
 	target.Stop()
 	ps.Stop()
 
diff --git a/pkg/promtail/targets/tailer.go b/pkg/promtail/targets/tailer.go
index 76da80356eacf..57ed49c672d23 100644
--- a/pkg/promtail/targets/tailer.go
+++ b/pkg/promtail/targets/tailer.go
@@ -48,6 +48,7 @@ func newTailer(logger log.Logger, handler api.EntryHandler, positions *positions
 
 	tail, err := tail.TailFile(filename, tail.Config{
 		Follow: true,
+		Poll:   true,
 		ReOpen: reOpen,
 		Location: &tail.SeekInfo{
 			Offset: positions.Get(filename),