From 77c4463d1515822b06b5f83dd10a883ac1c81380 Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Wed, 16 Feb 2022 18:24:24 -0500 Subject: [PATCH 1/5] try tweaking some times to reduce flakiness --- operator/builtin/input/file/rotation_test.go | 16 ++++++++-------- operator/flusher/flusher_test.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/operator/builtin/input/file/rotation_test.go b/operator/builtin/input/file/rotation_test.go index 5096267d1..1bcd8b3b8 100644 --- a/operator/builtin/input/file/rotation_test.go +++ b/operator/builtin/input/file/rotation_test.go @@ -285,7 +285,7 @@ func TestRotation(t *testing.T) { maxLinesPerFile: 10, maxBackupFiles: 1, writeInterval: time.Millisecond, - pollInterval: 20 * time.Millisecond, + pollInterval: 10 * time.Millisecond, }, { name: "Fast/NoDeletion", @@ -293,7 +293,7 @@ func TestRotation(t *testing.T) { maxLinesPerFile: 10, maxBackupFiles: 1, writeInterval: time.Millisecond, - pollInterval: 20 * time.Millisecond, + pollInterval: 10 * time.Millisecond, }, { name: "Fast/Deletion", @@ -301,7 +301,7 @@ func TestRotation(t *testing.T) { maxLinesPerFile: 10, maxBackupFiles: 1, writeInterval: time.Millisecond, - pollInterval: 20 * time.Millisecond, + pollInterval: 10 * time.Millisecond, ephemeralLines: true, }, { @@ -310,7 +310,7 @@ func TestRotation(t *testing.T) { maxLinesPerFile: 100, maxBackupFiles: 1, writeInterval: time.Millisecond, - pollInterval: 20 * time.Millisecond, + pollInterval: 10 * time.Millisecond, ephemeralLines: true, }, { @@ -319,7 +319,7 @@ func TestRotation(t *testing.T) { maxLinesPerFile: 10, maxBackupFiles: 1, writeInterval: 3 * time.Millisecond, - pollInterval: 20 * time.Millisecond, + pollInterval: 10 * time.Millisecond, }, { name: "Slow/NoDeletion", @@ -327,7 +327,7 @@ func TestRotation(t *testing.T) { maxLinesPerFile: 10, maxBackupFiles: 1, writeInterval: 3 * time.Millisecond, - pollInterval: 20 * time.Millisecond, + pollInterval: 10 * time.Millisecond, }, { name: "Slow/Deletion", @@ -335,7 +335,7 @@ func TestRotation(t *testing.T) { maxLinesPerFile: 10, maxBackupFiles: 1, writeInterval: 3 * time.Millisecond, - pollInterval: 20 * time.Millisecond, + pollInterval: 10 * time.Millisecond, }, { name: "Slow/Deletion/ExceedFingerprint", @@ -343,7 +343,7 @@ func TestRotation(t *testing.T) { maxLinesPerFile: 25, // ~20 is just enough to exceed 1000 bytes fingerprint at 50 chars per line maxBackupFiles: 2, writeInterval: 3 * time.Millisecond, - pollInterval: 20 * time.Millisecond, + pollInterval: 10 * time.Millisecond, }, } diff --git a/operator/flusher/flusher_test.go b/operator/flusher/flusher_test.go index c371da389..e50d199de 100644 --- a/operator/flusher/flusher_test.go +++ b/operator/flusher/flusher_test.go @@ -44,7 +44,7 @@ func TestFlusher(t *testing.T) { func TestMaxElapsedTime(t *testing.T) { // Override setting for test - maxElapsedTime = 100 * time.Millisecond + maxElapsedTime = 1 * time.Second flusherCfg := NewConfig() flusher := flusherCfg.Build(zaptest.NewLogger(t).Sugar()) From 463ea8445f9649f190d6f5c6a30568acc99de19d Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Wed, 16 Feb 2022 19:18:59 -0500 Subject: [PATCH 2/5] try upping wait interval when reading entries --- operator/builtin/input/file/rotation_test.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/operator/builtin/input/file/rotation_test.go b/operator/builtin/input/file/rotation_test.go index 1bcd8b3b8..507b10d14 100644 --- a/operator/builtin/input/file/rotation_test.go +++ b/operator/builtin/input/file/rotation_test.go @@ -190,6 +190,7 @@ type rotationTest struct { writeInterval time.Duration pollInterval time.Duration ephemeralLines bool + waitInterval time.Duration } /* @@ -259,7 +260,7 @@ func (rt rotationTest) run(tc rotationTest, copyTruncate, sequential bool) func( select { case e := <-logReceived: received = append(received, e.Record.(string)) - case <-time.After(100 * time.Millisecond): + case <-time.After(tc.waitInterval): break LOOP } } @@ -286,6 +287,7 @@ func TestRotation(t *testing.T) { maxBackupFiles: 1, writeInterval: time.Millisecond, pollInterval: 10 * time.Millisecond, + waitInterval: 500 * time.Millisecond, }, { name: "Fast/NoDeletion", @@ -294,6 +296,7 @@ func TestRotation(t *testing.T) { maxBackupFiles: 1, writeInterval: time.Millisecond, pollInterval: 10 * time.Millisecond, + waitInterval: 500 * time.Millisecond, }, { name: "Fast/Deletion", @@ -302,6 +305,7 @@ func TestRotation(t *testing.T) { maxBackupFiles: 1, writeInterval: time.Millisecond, pollInterval: 10 * time.Millisecond, + waitInterval: 500 * time.Millisecond, ephemeralLines: true, }, { @@ -311,6 +315,7 @@ func TestRotation(t *testing.T) { maxBackupFiles: 1, writeInterval: time.Millisecond, pollInterval: 10 * time.Millisecond, + waitInterval: 500 * time.Millisecond, ephemeralLines: true, }, { @@ -320,6 +325,7 @@ func TestRotation(t *testing.T) { maxBackupFiles: 1, writeInterval: 3 * time.Millisecond, pollInterval: 10 * time.Millisecond, + waitInterval: 500 * time.Millisecond, }, { name: "Slow/NoDeletion", @@ -328,6 +334,7 @@ func TestRotation(t *testing.T) { maxBackupFiles: 1, writeInterval: 3 * time.Millisecond, pollInterval: 10 * time.Millisecond, + waitInterval: 500 * time.Millisecond, }, { name: "Slow/Deletion", @@ -336,6 +343,7 @@ func TestRotation(t *testing.T) { maxBackupFiles: 1, writeInterval: 3 * time.Millisecond, pollInterval: 10 * time.Millisecond, + waitInterval: 500 * time.Millisecond, }, { name: "Slow/Deletion/ExceedFingerprint", @@ -344,6 +352,7 @@ func TestRotation(t *testing.T) { maxBackupFiles: 2, writeInterval: 3 * time.Millisecond, pollInterval: 10 * time.Millisecond, + waitInterval: 500 * time.Millisecond, }, } From 78744c6a0ee042a07fdbd36b3ffee898795de8ca Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Wed, 16 Feb 2022 22:34:44 -0500 Subject: [PATCH 3/5] close writer to force sync to disk in rotation tests --- operator/builtin/input/file/rotation_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/operator/builtin/input/file/rotation_test.go b/operator/builtin/input/file/rotation_test.go index 507b10d14..8596d8b85 100644 --- a/operator/builtin/input/file/rotation_test.go +++ b/operator/builtin/input/file/rotation_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/observiq/nanojack" "github.com/observiq/stanza/entry" "github.com/observiq/stanza/operator/helper" "github.com/observiq/stanza/testutil" @@ -254,6 +255,9 @@ func (rt rotationTest) run(tc rotationTest, copyTruncate, sequential bool) func( time.Sleep(tc.writeInterval) } + // Close the logger's writer to force a sync to disk + require.NoError(t, logger.Writer().(*nanojack.Logger).Close()) + received := make([]string, 0, tc.totalLines) LOOP: for { From 99ea77f7c35b0cec8b7c5c269cdfe7628e392626 Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Thu, 17 Feb 2022 14:35:36 -0500 Subject: [PATCH 4/5] try otel test cases --- operator/builtin/input/file/rotation_test.go | 56 ++------------------ 1 file changed, 5 insertions(+), 51 deletions(-) diff --git a/operator/builtin/input/file/rotation_test.go b/operator/builtin/input/file/rotation_test.go index 8596d8b85..fc7058332 100644 --- a/operator/builtin/input/file/rotation_test.go +++ b/operator/builtin/input/file/rotation_test.go @@ -11,7 +11,6 @@ import ( "testing" "time" - "github.com/observiq/nanojack" "github.com/observiq/stanza/entry" "github.com/observiq/stanza/operator/helper" "github.com/observiq/stanza/testutil" @@ -191,7 +190,6 @@ type rotationTest struct { writeInterval time.Duration pollInterval time.Duration ephemeralLines bool - waitInterval time.Duration } /* @@ -255,16 +253,13 @@ func (rt rotationTest) run(tc rotationTest, copyTruncate, sequential bool) func( time.Sleep(tc.writeInterval) } - // Close the logger's writer to force a sync to disk - require.NoError(t, logger.Writer().(*nanojack.Logger).Close()) - received := make([]string, 0, tc.totalLines) LOOP: for { select { case e := <-logReceived: received = append(received, e.Record.(string)) - case <-time.After(tc.waitInterval): + case <-time.After(200 * time.Millisecond): break LOOP } } @@ -282,82 +277,41 @@ func (rt rotationTest) run(tc rotationTest, copyTruncate, sequential bool) func( } func TestRotation(t *testing.T) { - cases := []rotationTest{ { - name: "Fast/NoRotation", + name: "NoRotation", totalLines: 10, maxLinesPerFile: 10, maxBackupFiles: 1, writeInterval: time.Millisecond, pollInterval: 10 * time.Millisecond, - waitInterval: 500 * time.Millisecond, }, { - name: "Fast/NoDeletion", + name: "NoDeletion", totalLines: 20, maxLinesPerFile: 10, maxBackupFiles: 1, writeInterval: time.Millisecond, pollInterval: 10 * time.Millisecond, - waitInterval: 500 * time.Millisecond, }, { - name: "Fast/Deletion", + name: "Deletion", totalLines: 30, maxLinesPerFile: 10, maxBackupFiles: 1, writeInterval: time.Millisecond, pollInterval: 10 * time.Millisecond, - waitInterval: 500 * time.Millisecond, ephemeralLines: true, }, { - name: "Fast/Deletion/ExceedFingerprint", + name: "Deletion/ExceedFingerprint", totalLines: 300, maxLinesPerFile: 100, maxBackupFiles: 1, writeInterval: time.Millisecond, pollInterval: 10 * time.Millisecond, - waitInterval: 500 * time.Millisecond, ephemeralLines: true, }, - { - name: "Slow/NoRotation", - totalLines: 10, - maxLinesPerFile: 10, - maxBackupFiles: 1, - writeInterval: 3 * time.Millisecond, - pollInterval: 10 * time.Millisecond, - waitInterval: 500 * time.Millisecond, - }, - { - name: "Slow/NoDeletion", - totalLines: 20, - maxLinesPerFile: 10, - maxBackupFiles: 1, - writeInterval: 3 * time.Millisecond, - pollInterval: 10 * time.Millisecond, - waitInterval: 500 * time.Millisecond, - }, - { - name: "Slow/Deletion", - totalLines: 30, - maxLinesPerFile: 10, - maxBackupFiles: 1, - writeInterval: 3 * time.Millisecond, - pollInterval: 10 * time.Millisecond, - waitInterval: 500 * time.Millisecond, - }, - { - name: "Slow/Deletion/ExceedFingerprint", - totalLines: 100, - maxLinesPerFile: 25, // ~20 is just enough to exceed 1000 bytes fingerprint at 50 chars per line - maxBackupFiles: 2, - writeInterval: 3 * time.Millisecond, - pollInterval: 10 * time.Millisecond, - waitInterval: 500 * time.Millisecond, - }, } for _, tc := range cases { From c97bd1a98bc0084c2105f2f1948a9487da55e328 Mon Sep 17 00:00:00 2001 From: Brandon Johnson Date: Mon, 21 Feb 2022 14:20:55 -0500 Subject: [PATCH 5/5] Equalize some of the timeouts --- operator/builtin/input/file/rotation_test.go | 2 +- operator/builtin/input/file/util_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/operator/builtin/input/file/rotation_test.go b/operator/builtin/input/file/rotation_test.go index fc7058332..9321b5434 100644 --- a/operator/builtin/input/file/rotation_test.go +++ b/operator/builtin/input/file/rotation_test.go @@ -259,7 +259,7 @@ func (rt rotationTest) run(tc rotationTest, copyTruncate, sequential bool) func( select { case e := <-logReceived: received = append(received, e.Record.(string)) - case <-time.After(200 * time.Millisecond): + case <-time.After(3 * time.Second): break LOOP } } diff --git a/operator/builtin/input/file/util_test.go b/operator/builtin/input/file/util_test.go index a124f7389..e126c9a97 100644 --- a/operator/builtin/input/file/util_test.go +++ b/operator/builtin/input/file/util_test.go @@ -143,7 +143,7 @@ LOOP: select { case e := <-c: receivedMessages = append(receivedMessages, e.Record.(string)) - case <-time.After(time.Second): + case <-time.After(3 * time.Second): break LOOP } } @@ -152,7 +152,7 @@ LOOP: } func expectNoMessages(t *testing.T, c chan *entry.Entry) { - expectNoMessagesUntil(t, c, 200*time.Millisecond) + expectNoMessagesUntil(t, c, 3*time.Second) } func expectNoMessagesUntil(t *testing.T, c chan *entry.Entry, d time.Duration) {