From 386b75850ea65f1864fe7e5c910058f8ac5302c8 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Thu, 13 Dec 2018 15:55:54 +0100 Subject: [PATCH 01/11] New: Impl. of non-transparent framing for telegraf syslog input plugin Modify syslog input plugin to support non-transparent framing technique (RFC6587), too. Modify other existing modalities to match new go-syslog interfaces. Signed-off-by: Leonardo Di Donato --- Gopkg.toml | 2 +- plugins/inputs/syslog/syslog.go | 85 ++++++++++++++++++++++----------- 2 files changed, 58 insertions(+), 29 deletions(-) diff --git a/Gopkg.toml b/Gopkg.toml index 80df324dcc0f5..366ec6db9f7c1 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -72,7 +72,7 @@ [[constraint]] name = "github.com/influxdata/go-syslog" - version = "1.0.1" + version = "feature/rfc6587" [[constraint]] name = "github.com/influxdata/tail" diff --git a/plugins/inputs/syslog/syslog.go b/plugins/inputs/syslog/syslog.go index 034e03df2d012..b4cb2bff541f5 100644 --- a/plugins/inputs/syslog/syslog.go +++ b/plugins/inputs/syslog/syslog.go @@ -12,8 +12,10 @@ import ( "time" "unicode" + "github.com/influxdata/go-syslog" "github.com/influxdata/go-syslog/rfc5424" "github.com/influxdata/go-syslog/rfc5425" + "github.com/influxdata/go-syslog/rfc6587" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" tlsConfig "github.com/influxdata/telegraf/internal/tls" @@ -26,12 +28,14 @@ const ipMaxPacketSize = 64 * 1024 // Syslog is a syslog plugin type Syslog struct { tlsConfig.ServerConfig - Address string `toml:"server"` - KeepAlivePeriod *internal.Duration - ReadTimeout *internal.Duration - MaxConnections int - BestEffort bool - Separator string `toml:"sdparam_separator"` + Address string `toml:"server"` + KeepAlivePeriod *internal.Duration + MaxConnections int + ReadTimeout *internal.Duration + TransparentFraming bool + Trailer rfc6587.TrailerType + BestEffort bool + Separator string `toml:"sdparam_separator"` now func() time.Time lastTime time.Time @@ -76,6 +80,15 @@ var sampleConfig = ` ## 0 means unlimited. # read_timeout = "5s" + ## Whether the messages come using the transparent framing or not (default = false). + ## When false messages come using non-transparent framing technique (RFC6587#section-3.4.2). + ## True means messages come using octect-counting framing technique (RFC5425#section-4.3.1). + # transparent_framing = false + + ## The trailer to be expected in case of non-trasparent framing (default = "LF"). + ## Must be one of "LF", or "NUL". + # trailer = "LF" + ## Whether to parse in best effort mode or not (default = false). ## By default best effort parsing is off. # best_effort = false @@ -95,7 +108,7 @@ func (s *Syslog) SampleConfig() string { // Description returns the plugin description func (s *Syslog) Description() string { - return "Accepts syslog messages per RFC5425" + return "Accepts syslog messages following RFC5424 format with transports as per RFC5426, RFC5425, or RFC6587" } // Gather ... @@ -203,7 +216,12 @@ func getAddressParts(a string) (string, string, error) { func (s *Syslog) listenPacket(acc telegraf.Accumulator) { defer s.wg.Done() b := make([]byte, ipMaxPacketSize) - p := rfc5424.NewParser() + var p syslog.Machine + if s.BestEffort { + p = rfc5424.NewParser(rfc5424.WithBestEffort()) + } else { + p = rfc5424.NewParser() + } for { n, _, err := s.udpListener.ReadFrom(b) if err != nil { @@ -213,9 +231,9 @@ func (s *Syslog) listenPacket(acc telegraf.Accumulator) { break } - message, err := p.Parse(b[:n], &s.BestEffort) + message, err := p.Parse(b[:n]) if message != nil { - acc.AddFields("syslog", fields(*message, s), tags(*message), s.time()) + acc.AddFields("syslog", fields(message, s), tags(message), s.time()) } if err != nil { acc.AddError(err) @@ -276,24 +294,38 @@ func (s *Syslog) handle(conn net.Conn, acc telegraf.Accumulator) { conn.Close() }() - var p *rfc5425.Parser + var p syslog.Parser + + emit := func(r *syslog.Result) { + s.store(*r, acc) + if s.ReadTimeout != nil && s.ReadTimeout.Duration > 0 { + conn.SetReadDeadline(time.Now().Add(s.ReadTimeout.Duration)) + } + } + // Create parser options + opts := []syslog.ParserOption{ + syslog.WithListener(emit), + } if s.BestEffort { - p = rfc5425.NewParser(conn, rfc5425.WithBestEffort()) + opts = append(opts, syslog.WithBestEffort()) + } + + // Select the parser to use depeding on transport framing + if s.TransparentFraming { + // Octet-counting transparent framing + p = rfc5425.NewParser(opts...) } else { - p = rfc5425.NewParser(conn) + // Non-transparent framing + opts = append(opts, rfc6587.WithTrailer(s.Trailer)) + p = rfc6587.NewParser(opts...) } + p.Parse(conn) + if s.ReadTimeout != nil && s.ReadTimeout.Duration > 0 { conn.SetReadDeadline(time.Now().Add(s.ReadTimeout.Duration)) } - - p.ParseExecuting(func(r *rfc5425.Result) { - s.store(*r, acc) - if s.ReadTimeout != nil && s.ReadTimeout.Duration > 0 { - conn.SetReadDeadline(time.Now().Add(s.ReadTimeout.Duration)) - } - }) } func (s *Syslog) setKeepAlive(c *net.TCPConn) error { @@ -310,20 +342,16 @@ func (s *Syslog) setKeepAlive(c *net.TCPConn) error { return c.SetKeepAlivePeriod(s.KeepAlivePeriod.Duration) } -func (s *Syslog) store(res rfc5425.Result, acc telegraf.Accumulator) { +func (s *Syslog) store(res syslog.Result, acc telegraf.Accumulator) { if res.Error != nil { acc.AddError(res.Error) } - if res.MessageError != nil { - acc.AddError(res.MessageError) - } if res.Message != nil { - msg := *res.Message - acc.AddFields("syslog", fields(msg, s), tags(msg), s.time()) + acc.AddFields("syslog", fields(res.Message, s), tags(res.Message), s.time()) } } -func tags(msg rfc5424.SyslogMessage) map[string]string { +func tags(msg syslog.Message) map[string]string { ts := map[string]string{} // Not checking assuming a minimally valid message @@ -341,7 +369,7 @@ func tags(msg rfc5424.SyslogMessage) map[string]string { return ts } -func fields(msg rfc5424.SyslogMessage, s *Syslog) map[string]interface{} { +func fields(msg syslog.Message, s *Syslog) map[string]interface{} { // Not checking assuming a minimally valid message flds := map[string]interface{}{ "version": msg.Version(), @@ -415,6 +443,7 @@ func init() { ReadTimeout: &internal.Duration{ Duration: defaultReadTimeout, }, + Trailer: rfc6587.LF, Separator: "_", } From e9f2393d856356b740d9fb6158f0a0260d53763e Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Thu, 13 Dec 2018 16:59:18 +0100 Subject: [PATCH 02/11] Update: Transparent framing is not the default anymore RFC5425 tests need to be updated accordingly. Signed-off-by: Leonardo Di Donato --- plugins/inputs/syslog/rfc5425_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/inputs/syslog/rfc5425_test.go b/plugins/inputs/syslog/rfc5425_test.go index d629024b79151..eda38bb5df5b8 100644 --- a/plugins/inputs/syslog/rfc5425_test.go +++ b/plugins/inputs/syslog/rfc5425_test.go @@ -355,9 +355,10 @@ func newTCPSyslogReceiver(address string, keepAlive *internal.Duration, maxConn now: func() time.Time { return defaultTime }, - ReadTimeout: d, - BestEffort: bestEffort, - Separator: "_", + TransparentFraming: true, + ReadTimeout: d, + BestEffort: bestEffort, + Separator: "_", } if keepAlive != nil { s.KeepAlivePeriod = keepAlive From 57546c5c1c166026504a5e87b6448f260cc7cdb0 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 14 Dec 2018 11:44:53 +0100 Subject: [PATCH 03/11] Update: Move receivers and test structs away to prepare new test cases for RFC6587 --- plugins/inputs/syslog/commons_test.go | 62 ++++++++++++++++++++++++++ plugins/inputs/syslog/rfc5425_test.go | 64 ++++++--------------------- plugins/inputs/syslog/rfc5426_test.go | 23 +--------- 3 files changed, 78 insertions(+), 71 deletions(-) create mode 100644 plugins/inputs/syslog/commons_test.go diff --git a/plugins/inputs/syslog/commons_test.go b/plugins/inputs/syslog/commons_test.go new file mode 100644 index 0000000000000..c21b742a3072a --- /dev/null +++ b/plugins/inputs/syslog/commons_test.go @@ -0,0 +1,62 @@ +package syslog + +import ( + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/testutil" + "time" +) + +var ( + pki = testutil.NewPKI("../../../testutil/pki") +) + +type testCasePacket struct { + name string + data []byte + wantBestEffort *testutil.Metric + wantStrict *testutil.Metric + werr bool +} + +type testCaseStream struct { + name string + data []byte + wantBestEffort []testutil.Metric + wantStrict []testutil.Metric + werr int // how many errors we expect in the strict mode? +} + +func newUDPSyslogReceiver(address string, bestEffort bool) *Syslog { + return &Syslog{ + Address: address, + now: func() time.Time { + return defaultTime + }, + BestEffort: bestEffort, + Separator: "_", + } +} + +func newTCPSyslogReceiver(address string, keepAlive *internal.Duration, maxConn int, bestEffort bool, transparent bool) *Syslog { + d := &internal.Duration{ + Duration: defaultReadTimeout, + } + s := &Syslog{ + Address: address, + now: func() time.Time { + return defaultTime + }, + TransparentFraming: transparent, + ReadTimeout: d, + BestEffort: bestEffort, + Separator: "_", + } + if keepAlive != nil { + s.KeepAlivePeriod = keepAlive + } + if maxConn > 0 { + s.MaxConnections = maxConn + } + + return s +} diff --git a/plugins/inputs/syslog/rfc5425_test.go b/plugins/inputs/syslog/rfc5425_test.go index eda38bb5df5b8..c19c7f6b226a3 100644 --- a/plugins/inputs/syslog/rfc5425_test.go +++ b/plugins/inputs/syslog/rfc5425_test.go @@ -16,20 +16,8 @@ import ( "github.com/stretchr/testify/require" ) -var ( - pki = testutil.NewPKI("../../../testutil/pki") -) - -type testCase5425 struct { - name string - data []byte - wantBestEffort []testutil.Metric - wantStrict []testutil.Metric - werr int // how many errors we expect in the strict mode? -} - -func getTestCasesForRFC5425() []testCase5425 { - testCases := []testCase5425{ +func getTestCasesForRFC5425() []testCaseStream { + testCases := []testCaseStream{ { name: "1st/avg/ok", data: []byte(`188 <29>1 2016-02-21T04:32:57+00:00 web1 someservice 2341 2 [origin][meta sequence="14125553" service="someservice"] "GET /v1/ok HTTP/1.1" 200 145 "-" "hacheck 0.9.0" 24306 127.0.0.1:40124 575`), @@ -346,35 +334,11 @@ func getTestCasesForRFC5425() []testCase5425 { return testCases } -func newTCPSyslogReceiver(address string, keepAlive *internal.Duration, maxConn int, bestEffort bool) *Syslog { - d := &internal.Duration{ - Duration: defaultReadTimeout, - } - s := &Syslog{ - Address: address, - now: func() time.Time { - return defaultTime - }, - TransparentFraming: true, - ReadTimeout: d, - BestEffort: bestEffort, - Separator: "_", - } - if keepAlive != nil { - s.KeepAlivePeriod = keepAlive - } - if maxConn > 0 { - s.MaxConnections = maxConn - } - - return s -} - func testStrictRFC5425(t *testing.T, protocol string, address string, wantTLS bool, keepAlive *internal.Duration) { for _, tc := range getTestCasesForRFC5425() { t.Run(tc.name, func(t *testing.T) { // Creation of a strict mode receiver - receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, false) + receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, false, true) require.NotNil(t, receiver) if wantTLS { receiver.ServerConfig = *pki.TLSServerConfig() @@ -436,7 +400,7 @@ func testBestEffortRFC5425(t *testing.T, protocol string, address string, wantTL for _, tc := range getTestCasesForRFC5425() { t.Run(tc.name, func(t *testing.T) { // Creation of a best effort mode receiver - receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, true) + receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, true, true) require.NotNil(t, receiver) if wantTLS { receiver.ServerConfig = *pki.TLSServerConfig() @@ -487,31 +451,31 @@ func testBestEffortRFC5425(t *testing.T, protocol string, address string, wantTL } } -func TestStrict_tcp(t *testing.T) { +func TestRFC5425Strict_tcp(t *testing.T) { testStrictRFC5425(t, "tcp", address, false, nil) } -func TestBestEffort_tcp(t *testing.T) { +func TestRFC5425BestEffort_tcp(t *testing.T) { testBestEffortRFC5425(t, "tcp", address, false, nil) } -func TestStrict_tcp_tls(t *testing.T) { +func TestRFC5425Strict_tcp_tls(t *testing.T) { testStrictRFC5425(t, "tcp", address, true, nil) } -func TestBestEffort_tcp_tls(t *testing.T) { +func TestRFC5425BestEffort_tcp_tls(t *testing.T) { testBestEffortRFC5425(t, "tcp", address, true, nil) } -func TestStrictWithKeepAlive_tcp_tls(t *testing.T) { +func TestRFC5425StrictWithKeepAlive_tcp_tls(t *testing.T) { testStrictRFC5425(t, "tcp", address, true, &internal.Duration{Duration: time.Minute}) } -func TestStrictWithZeroKeepAlive_tcp_tls(t *testing.T) { +func TestRFC5425StrictWithZeroKeepAlive_tcp_tls(t *testing.T) { testStrictRFC5425(t, "tcp", address, true, &internal.Duration{Duration: 0}) } -func TestStrict_unix(t *testing.T) { +func TestRFC5425Strict_unix(t *testing.T) { tmpdir, err := ioutil.TempDir("", "telegraf") require.NoError(t, err) defer os.RemoveAll(tmpdir) @@ -519,7 +483,7 @@ func TestStrict_unix(t *testing.T) { testStrictRFC5425(t, "unix", sock, false, nil) } -func TestBestEffort_unix(t *testing.T) { +func TestRFC5425BestEffort_unix(t *testing.T) { tmpdir, err := ioutil.TempDir("", "telegraf") require.NoError(t, err) defer os.RemoveAll(tmpdir) @@ -527,7 +491,7 @@ func TestBestEffort_unix(t *testing.T) { testBestEffortRFC5425(t, "unix", sock, false, nil) } -func TestStrict_unix_tls(t *testing.T) { +func TestRFC5425Strict_unix_tls(t *testing.T) { tmpdir, err := ioutil.TempDir("", "telegraf") require.NoError(t, err) defer os.RemoveAll(tmpdir) @@ -535,7 +499,7 @@ func TestStrict_unix_tls(t *testing.T) { testStrictRFC5425(t, "unix", sock, true, nil) } -func TestBestEffort_unix_tls(t *testing.T) { +func TestRFC5425BestEffort_unix_tls(t *testing.T) { tmpdir, err := ioutil.TempDir("", "telegraf") require.NoError(t, err) defer os.RemoveAll(tmpdir) diff --git a/plugins/inputs/syslog/rfc5426_test.go b/plugins/inputs/syslog/rfc5426_test.go index 67966ed1de406..ba856b0ac2bc8 100644 --- a/plugins/inputs/syslog/rfc5426_test.go +++ b/plugins/inputs/syslog/rfc5426_test.go @@ -15,16 +15,8 @@ import ( "github.com/stretchr/testify/require" ) -type testCase5426 struct { - name string - data []byte - wantBestEffort *testutil.Metric - wantStrict *testutil.Metric - werr bool -} - -func getTestCasesForRFC5426() []testCase5426 { - testCases := []testCase5426{ +func getTestCasesForRFC5426() []testCasePacket { + testCases := []testCasePacket{ { name: "empty", data: []byte(""), @@ -239,17 +231,6 @@ func getTestCasesForRFC5426() []testCase5426 { return testCases } -func newUDPSyslogReceiver(address string, bestEffort bool) *Syslog { - return &Syslog{ - Address: address, - now: func() time.Time { - return defaultTime - }, - BestEffort: bestEffort, - Separator: "_", - } -} - func testRFC5426(t *testing.T, protocol string, address string, bestEffort bool) { for _, tc := range getTestCasesForRFC5426() { t.Run(tc.name, func(t *testing.T) { From dab9cc6ea9d73610728037d946c8bf7dda7106e4 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 14 Dec 2018 11:45:45 +0100 Subject: [PATCH 04/11] New: Initial test for non-transparent framing of syslog messages 1 test case, all combinations. Signed-off-by: Leonardo Di Donato --- plugins/inputs/syslog/rfc6587_test.go | 247 ++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 plugins/inputs/syslog/rfc6587_test.go diff --git a/plugins/inputs/syslog/rfc6587_test.go b/plugins/inputs/syslog/rfc6587_test.go new file mode 100644 index 0000000000000..20ef3fdd4d193 --- /dev/null +++ b/plugins/inputs/syslog/rfc6587_test.go @@ -0,0 +1,247 @@ +package syslog + +import ( + "crypto/tls" + "io/ioutil" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func getTestCasesForRFC6587() []testCaseStream { + testCases := []testCaseStream{ + { + name: "1st/avg/ok", + data: []byte(`<29>1 2016-02-21T04:32:57+00:00 web1 someservice 2341 2 [origin][meta sequence="14125553" service="someservice"] "GET /v1/ok HTTP/1.1" 200 145 "-" "hacheck 0.9.0" 24306 127.0.0.1:40124 575`), + wantStrict: []testutil.Metric{ + { + Measurement: "syslog", + Fields: map[string]interface{}{ + "version": uint16(1), + "timestamp": time.Unix(1456029177, 0).UnixNano(), + "procid": "2341", + "msgid": "2", + "message": `"GET /v1/ok HTTP/1.1" 200 145 "-" "hacheck 0.9.0" 24306 127.0.0.1:40124 575`, + "origin": true, + "meta_sequence": "14125553", + "meta_service": "someservice", + "severity_code": 5, + "facility_code": 3, + }, + Tags: map[string]string{ + "severity": "notice", + "facility": "daemon", + "hostname": "web1", + "appname": "someservice", + }, + Time: defaultTime, + }, + }, + wantBestEffort: []testutil.Metric{ + { + Measurement: "syslog", + Fields: map[string]interface{}{ + "version": uint16(1), + "timestamp": time.Unix(1456029177, 0).UnixNano(), + "procid": "2341", + "msgid": "2", + "message": `"GET /v1/ok HTTP/1.1" 200 145 "-" "hacheck 0.9.0" 24306 127.0.0.1:40124 575`, + "origin": true, + "meta_sequence": "14125553", + "meta_service": "someservice", + "severity_code": 5, + "facility_code": 3, + }, + Tags: map[string]string{ + "severity": "notice", + "facility": "daemon", + "hostname": "web1", + "appname": "someservice", + }, + Time: defaultTime, + }, + }, + }, + } + return testCases +} + +func testStrictRFC6587(t *testing.T, protocol string, address string, wantTLS bool, keepAlive *internal.Duration) { + for _, tc := range getTestCasesForRFC6587() { + t.Run(tc.name, func(t *testing.T) { + // Creation of a strict mode receiver + receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, false, false) + require.NotNil(t, receiver) + if wantTLS { + receiver.ServerConfig = *pki.TLSServerConfig() + } + require.Equal(t, receiver.KeepAlivePeriod, keepAlive) + acc := &testutil.Accumulator{} + require.NoError(t, receiver.Start(acc)) + defer receiver.Stop() + + // Connect + var conn net.Conn + var err error + if wantTLS { + config, e := pki.TLSClientConfig().TLSConfig() + require.NoError(t, e) + config.ServerName = "localhost" + conn, err = tls.Dial(protocol, address, config) + } else { + conn, err = net.Dial(protocol, address) + defer conn.Close() + } + require.NotNil(t, conn) + require.NoError(t, err) + + // Clear + acc.ClearMetrics() + acc.Errors = make([]error, 0) + + // Write + _, err = conn.Write(tc.data) + conn.Close() + require.NoError(t, err) + + // Wait that the the number of data points is accumulated + // Since the receiver is running concurrently + if tc.wantStrict != nil { + acc.Wait(len(tc.wantStrict)) + } + + // Wait the parsing error + acc.WaitError(tc.werr) + + // Verify + if len(acc.Errors) != tc.werr { + t.Fatalf("Got unexpected errors. want error = %v, errors = %v\n", tc.werr, acc.Errors) + } + var got []testutil.Metric + for _, metric := range acc.Metrics { + got = append(got, *metric) + } + if !cmp.Equal(tc.wantStrict, got) { + t.Fatalf("Got (+) / Want (-)\n %s", cmp.Diff(tc.wantStrict, got)) + } + }) + } +} + +func testBestEffortRFC6587(t *testing.T, protocol string, address string, wantTLS bool, keepAlive *internal.Duration) { + for _, tc := range getTestCasesForRFC6587() { + t.Run(tc.name, func(t *testing.T) { + // Creation of a best effort mode receiver + receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, true, false) + require.NotNil(t, receiver) + if wantTLS { + receiver.ServerConfig = *pki.TLSServerConfig() + } + require.Equal(t, receiver.KeepAlivePeriod, keepAlive) + acc := &testutil.Accumulator{} + require.NoError(t, receiver.Start(acc)) + defer receiver.Stop() + + // Connect + var conn net.Conn + var err error + if wantTLS { + config, e := pki.TLSClientConfig().TLSConfig() + require.NoError(t, e) + config.ServerName = "localhost" + conn, err = tls.Dial(protocol, address, config) + } else { + conn, err = net.Dial(protocol, address) + } + require.NotNil(t, conn) + require.NoError(t, err) + + // Clear + acc.ClearMetrics() + acc.Errors = make([]error, 0) + + // Write + _, err = conn.Write(tc.data) + require.NoError(t, err) + conn.Close() + + // Wait that the the number of data points is accumulated + // Since the receiver is running concurrently + if tc.wantBestEffort != nil { + acc.Wait(len(tc.wantBestEffort)) + } + + // Verify + var got []testutil.Metric + for _, metric := range acc.Metrics { + got = append(got, *metric) + } + if !cmp.Equal(tc.wantBestEffort, got) { + t.Fatalf("Got (+) / Want (-)\n %s", cmp.Diff(tc.wantBestEffort, got)) + } + }) + } +} + +func TestRFC6587Strict_tcp(t *testing.T) { + testStrictRFC6587(t, "tcp", address, false, nil) +} + +func TestRFC6587BestEffort_tcp(t *testing.T) { + testBestEffortRFC6587(t, "tcp", address, false, nil) +} + +func TestRFC6587Strict_tcp_tls(t *testing.T) { + testStrictRFC6587(t, "tcp", address, true, nil) +} + +func TestRFC6587BestEffort_tcp_tls(t *testing.T) { + testBestEffortRFC6587(t, "tcp", address, true, nil) +} + +func TestRFC6587StrictWithKeepAlive_tcp_tls(t *testing.T) { + testStrictRFC6587(t, "tcp", address, true, &internal.Duration{Duration: time.Minute}) +} + +func TestRFC6587StrictWithZeroKeepAlive_tcp_tls(t *testing.T) { + testStrictRFC6587(t, "tcp", address, true, &internal.Duration{Duration: 0}) +} + +func TestRFC6587Strict_unix(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "telegraf") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + sock := filepath.Join(tmpdir, "syslog.TestStrict_unix.sock") + testStrictRFC6587(t, "unix", sock, false, nil) +} + +func TestRFC6587BestEffort_unix(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "telegraf") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + sock := filepath.Join(tmpdir, "syslog.TestBestEffort_unix.sock") + testBestEffortRFC6587(t, "unix", sock, false, nil) +} + +func TestRFC6587Strict_unix_tls(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "telegraf") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + sock := filepath.Join(tmpdir, "syslog.TestStrict_unix_tls.sock") + testStrictRFC6587(t, "unix", sock, true, nil) +} + +func TestRFC6587BestEffort_unix_tls(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "telegraf") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + sock := filepath.Join(tmpdir, "syslog.TestBestEffort_unix_tls.sock") + testBestEffortRFC6587(t, "unix", sock, true, nil) +} From f5adb227658174334a86be1bde2f7f0eb1326f5d Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 14 Dec 2018 16:26:44 +0100 Subject: [PATCH 05/11] Docs: Syslog input plugin transport options --- plugins/inputs/syslog/README.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/syslog/README.md b/plugins/inputs/syslog/README.md index ad9b9b57258ca..06693bf84473b 100644 --- a/plugins/inputs/syslog/README.md +++ b/plugins/inputs/syslog/README.md @@ -37,6 +37,15 @@ Syslog messages should be formatted according to ## 0 means unlimited. # read_timeout = "5s" + ## Whether the messages come using the transparent framing or not (default = false). + ## When false messages come using non-transparent framing technique (RFC6587#section-3.4.2). + ## True means messages come using octect-counting framing technique (RFC5425#section-4.3.1 and RFC6587#section-3.4.1). + # transparent_framing = false + + ## The trailer to be expected in case of non-trasparent framing (default = "LF"). + ## Must be one of "LF", or "NUL". + # trailer = "LF" + ## Whether to parse in best effort mode or not (default = false). ## By default best effort parsing is off. # best_effort = false @@ -49,11 +58,18 @@ Syslog messages should be formatted according to # sdparam_separator = "_" ``` -#### Best Effort +#### Message transport + +The `transparent_framing` option only applies to streams. It governs the way we expect to receive messages within the stream. +With the [octet counting](https://tools.ietf.org/html/rfc5425#section-4.3) technique or with the [non-transparent](https://tools.ietf.org/html/rfc6587#section-3.4.2) framing. + +The `trailer` option only applies when `transparent_framing` is `false` - ie., non-transparent transport. + +#### Best effort The [`best_effort`](https://github.com/influxdata/go-syslog#best-effort-mode) option instructs the parser to extract partial but valid info from syslog -messages. If unset only full messages will be collected. +messages. If unset only full messages will be collected. #### Rsyslog Integration From 8e9343a723457abcbbc145d90a2720297aa27290 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 14 Dec 2018 16:27:07 +0100 Subject: [PATCH 06/11] New: Tests for non transparent transport of syslog messages --- plugins/inputs/syslog/README.md | 3 +- ...rfc6587_test.go => nontransparent_test.go} | 111 ++++++++++++++---- ...{rfc5425_test.go => octetcounting_test.go} | 50 ++++---- plugins/inputs/syslog/syslog.go | 18 +-- 4 files changed, 122 insertions(+), 60 deletions(-) rename plugins/inputs/syslog/{rfc6587_test.go => nontransparent_test.go} (65%) rename plugins/inputs/syslog/{rfc5425_test.go => octetcounting_test.go} (88%) diff --git a/plugins/inputs/syslog/README.md b/plugins/inputs/syslog/README.md index 06693bf84473b..68018cda529a4 100644 --- a/plugins/inputs/syslog/README.md +++ b/plugins/inputs/syslog/README.md @@ -2,7 +2,8 @@ The syslog plugin listens for syslog messages transmitted over [UDP](https://tools.ietf.org/html/rfc5426) or -[TCP](https://tools.ietf.org/html/rfc5425). +[TCP](https://tools.ietf.org/html/rfc6587) or +[TLS](https://tools.ietf.org/html/rfc5425), with or without the octet couting framing. Syslog messages should be formatted according to [RFC 5424](https://tools.ietf.org/html/rfc5424). diff --git a/plugins/inputs/syslog/rfc6587_test.go b/plugins/inputs/syslog/nontransparent_test.go similarity index 65% rename from plugins/inputs/syslog/rfc6587_test.go rename to plugins/inputs/syslog/nontransparent_test.go index 20ef3fdd4d193..b3dbbc4e0e48a 100644 --- a/plugins/inputs/syslog/rfc6587_test.go +++ b/plugins/inputs/syslog/nontransparent_test.go @@ -15,7 +15,7 @@ import ( "github.com/stretchr/testify/require" ) -func getTestCasesForRFC6587() []testCaseStream { +func getTestCasesForNonTransparent() []testCaseStream { testCases := []testCaseStream{ { name: "1st/avg/ok", @@ -68,13 +68,74 @@ func getTestCasesForRFC6587() []testCaseStream { Time: defaultTime, }, }, + werr: 1, + }, + { + name: "1st/min/ok//2nd/min/ok", + data: []byte("<1>2 - - - - - -\n<4>11 - - - - - -\n"), + wantStrict: []testutil.Metric{ + { + Measurement: "syslog", + Fields: map[string]interface{}{ + "version": uint16(2), + "severity_code": 1, + "facility_code": 0, + }, + Tags: map[string]string{ + "severity": "alert", + "facility": "kern", + }, + Time: defaultTime, + }, + { + Measurement: "syslog", + Fields: map[string]interface{}{ + "version": uint16(11), + "severity_code": 4, + "facility_code": 0, + }, + Tags: map[string]string{ + "severity": "warning", + "facility": "kern", + }, + Time: defaultTime.Add(time.Nanosecond), + }, + }, + wantBestEffort: []testutil.Metric{ + { + Measurement: "syslog", + Fields: map[string]interface{}{ + "version": uint16(2), + "severity_code": 1, + "facility_code": 0, + }, + Tags: map[string]string{ + "severity": "alert", + "facility": "kern", + }, + Time: defaultTime, + }, + { + Measurement: "syslog", + Fields: map[string]interface{}{ + "version": uint16(11), + "severity_code": 4, + "facility_code": 0, + }, + Tags: map[string]string{ + "severity": "warning", + "facility": "kern", + }, + Time: defaultTime.Add(time.Nanosecond), + }, + }, }, } return testCases } -func testStrictRFC6587(t *testing.T, protocol string, address string, wantTLS bool, keepAlive *internal.Duration) { - for _, tc := range getTestCasesForRFC6587() { +func testStrictNonTransparent(t *testing.T, protocol string, address string, wantTLS bool, keepAlive *internal.Duration) { + for _, tc := range getTestCasesForNonTransparent() { t.Run(tc.name, func(t *testing.T) { // Creation of a strict mode receiver receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, false, false) @@ -135,8 +196,8 @@ func testStrictRFC6587(t *testing.T, protocol string, address string, wantTLS bo } } -func testBestEffortRFC6587(t *testing.T, protocol string, address string, wantTLS bool, keepAlive *internal.Duration) { - for _, tc := range getTestCasesForRFC6587() { +func testBestEffortNonTransparent(t *testing.T, protocol string, address string, wantTLS bool, keepAlive *internal.Duration) { + for _, tc := range getTestCasesForNonTransparent() { t.Run(tc.name, func(t *testing.T) { // Creation of a best effort mode receiver receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, true, false) @@ -190,58 +251,58 @@ func testBestEffortRFC6587(t *testing.T, protocol string, address string, wantTL } } -func TestRFC6587Strict_tcp(t *testing.T) { - testStrictRFC6587(t, "tcp", address, false, nil) +func TestNonTransparentStrict_tcp(t *testing.T) { + testStrictNonTransparent(t, "tcp", address, false, nil) } -func TestRFC6587BestEffort_tcp(t *testing.T) { - testBestEffortRFC6587(t, "tcp", address, false, nil) +func TestNonTransparentBestEffort_tcp(t *testing.T) { + testBestEffortNonTransparent(t, "tcp", address, false, nil) } -func TestRFC6587Strict_tcp_tls(t *testing.T) { - testStrictRFC6587(t, "tcp", address, true, nil) +func TestNonTransparentStrict_tcp_tls(t *testing.T) { + testStrictNonTransparent(t, "tcp", address, true, nil) } -func TestRFC6587BestEffort_tcp_tls(t *testing.T) { - testBestEffortRFC6587(t, "tcp", address, true, nil) +func TestNonTransparentBestEffort_tcp_tls(t *testing.T) { + testBestEffortNonTransparent(t, "tcp", address, true, nil) } -func TestRFC6587StrictWithKeepAlive_tcp_tls(t *testing.T) { - testStrictRFC6587(t, "tcp", address, true, &internal.Duration{Duration: time.Minute}) +func TestNonTransparentStrictWithKeepAlive_tcp_tls(t *testing.T) { + testStrictNonTransparent(t, "tcp", address, true, &internal.Duration{Duration: time.Minute}) } -func TestRFC6587StrictWithZeroKeepAlive_tcp_tls(t *testing.T) { - testStrictRFC6587(t, "tcp", address, true, &internal.Duration{Duration: 0}) +func TestNonTransparentStrictWithZeroKeepAlive_tcp_tls(t *testing.T) { + testStrictNonTransparent(t, "tcp", address, true, &internal.Duration{Duration: 0}) } -func TestRFC6587Strict_unix(t *testing.T) { +func TestNonTransparentStrict_unix(t *testing.T) { tmpdir, err := ioutil.TempDir("", "telegraf") require.NoError(t, err) defer os.RemoveAll(tmpdir) sock := filepath.Join(tmpdir, "syslog.TestStrict_unix.sock") - testStrictRFC6587(t, "unix", sock, false, nil) + testStrictNonTransparent(t, "unix", sock, false, nil) } -func TestRFC6587BestEffort_unix(t *testing.T) { +func TestNonTransparentBestEffort_unix(t *testing.T) { tmpdir, err := ioutil.TempDir("", "telegraf") require.NoError(t, err) defer os.RemoveAll(tmpdir) sock := filepath.Join(tmpdir, "syslog.TestBestEffort_unix.sock") - testBestEffortRFC6587(t, "unix", sock, false, nil) + testBestEffortNonTransparent(t, "unix", sock, false, nil) } -func TestRFC6587Strict_unix_tls(t *testing.T) { +func TestNonTransparentStrict_unix_tls(t *testing.T) { tmpdir, err := ioutil.TempDir("", "telegraf") require.NoError(t, err) defer os.RemoveAll(tmpdir) sock := filepath.Join(tmpdir, "syslog.TestStrict_unix_tls.sock") - testStrictRFC6587(t, "unix", sock, true, nil) + testStrictNonTransparent(t, "unix", sock, true, nil) } -func TestRFC6587BestEffort_unix_tls(t *testing.T) { +func TestNonTransparentBestEffort_unix_tls(t *testing.T) { tmpdir, err := ioutil.TempDir("", "telegraf") require.NoError(t, err) defer os.RemoveAll(tmpdir) sock := filepath.Join(tmpdir, "syslog.TestBestEffort_unix_tls.sock") - testBestEffortRFC6587(t, "unix", sock, true, nil) + testBestEffortNonTransparent(t, "unix", sock, true, nil) } diff --git a/plugins/inputs/syslog/rfc5425_test.go b/plugins/inputs/syslog/octetcounting_test.go similarity index 88% rename from plugins/inputs/syslog/rfc5425_test.go rename to plugins/inputs/syslog/octetcounting_test.go index c19c7f6b226a3..77f416b6bd9e2 100644 --- a/plugins/inputs/syslog/rfc5425_test.go +++ b/plugins/inputs/syslog/octetcounting_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/require" ) -func getTestCasesForRFC5425() []testCaseStream { +func getTestCasesForOctetCounting() []testCaseStream { testCases := []testCaseStream{ { name: "1st/avg/ok", @@ -334,8 +334,8 @@ func getTestCasesForRFC5425() []testCaseStream { return testCases } -func testStrictRFC5425(t *testing.T, protocol string, address string, wantTLS bool, keepAlive *internal.Duration) { - for _, tc := range getTestCasesForRFC5425() { +func testStrictOctetCounting(t *testing.T, protocol string, address string, wantTLS bool, keepAlive *internal.Duration) { + for _, tc := range getTestCasesForOctetCounting() { t.Run(tc.name, func(t *testing.T) { // Creation of a strict mode receiver receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, false, true) @@ -396,8 +396,8 @@ func testStrictRFC5425(t *testing.T, protocol string, address string, wantTLS bo } } -func testBestEffortRFC5425(t *testing.T, protocol string, address string, wantTLS bool, keepAlive *internal.Duration) { - for _, tc := range getTestCasesForRFC5425() { +func testBestEffortOctetCounting(t *testing.T, protocol string, address string, wantTLS bool, keepAlive *internal.Duration) { + for _, tc := range getTestCasesForOctetCounting() { t.Run(tc.name, func(t *testing.T) { // Creation of a best effort mode receiver receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, true, true) @@ -451,58 +451,58 @@ func testBestEffortRFC5425(t *testing.T, protocol string, address string, wantTL } } -func TestRFC5425Strict_tcp(t *testing.T) { - testStrictRFC5425(t, "tcp", address, false, nil) +func TestOctetCountingStrict_tcp(t *testing.T) { + testStrictOctetCounting(t, "tcp", address, false, nil) } -func TestRFC5425BestEffort_tcp(t *testing.T) { - testBestEffortRFC5425(t, "tcp", address, false, nil) +func TestOctetCountingBestEffort_tcp(t *testing.T) { + testBestEffortOctetCounting(t, "tcp", address, false, nil) } -func TestRFC5425Strict_tcp_tls(t *testing.T) { - testStrictRFC5425(t, "tcp", address, true, nil) +func TestOctetCountingStrict_tcp_tls(t *testing.T) { + testStrictOctetCounting(t, "tcp", address, true, nil) } -func TestRFC5425BestEffort_tcp_tls(t *testing.T) { - testBestEffortRFC5425(t, "tcp", address, true, nil) +func TestOctetCountingBestEffort_tcp_tls(t *testing.T) { + testBestEffortOctetCounting(t, "tcp", address, true, nil) } -func TestRFC5425StrictWithKeepAlive_tcp_tls(t *testing.T) { - testStrictRFC5425(t, "tcp", address, true, &internal.Duration{Duration: time.Minute}) +func TestOctetCountingStrictWithKeepAlive_tcp_tls(t *testing.T) { + testStrictOctetCounting(t, "tcp", address, true, &internal.Duration{Duration: time.Minute}) } -func TestRFC5425StrictWithZeroKeepAlive_tcp_tls(t *testing.T) { - testStrictRFC5425(t, "tcp", address, true, &internal.Duration{Duration: 0}) +func TestOctetCountingStrictWithZeroKeepAlive_tcp_tls(t *testing.T) { + testStrictOctetCounting(t, "tcp", address, true, &internal.Duration{Duration: 0}) } -func TestRFC5425Strict_unix(t *testing.T) { +func TestOctetCountingStrict_unix(t *testing.T) { tmpdir, err := ioutil.TempDir("", "telegraf") require.NoError(t, err) defer os.RemoveAll(tmpdir) sock := filepath.Join(tmpdir, "syslog.TestStrict_unix.sock") - testStrictRFC5425(t, "unix", sock, false, nil) + testStrictOctetCounting(t, "unix", sock, false, nil) } -func TestRFC5425BestEffort_unix(t *testing.T) { +func TestOctetCountingBestEffort_unix(t *testing.T) { tmpdir, err := ioutil.TempDir("", "telegraf") require.NoError(t, err) defer os.RemoveAll(tmpdir) sock := filepath.Join(tmpdir, "syslog.TestBestEffort_unix.sock") - testBestEffortRFC5425(t, "unix", sock, false, nil) + testBestEffortOctetCounting(t, "unix", sock, false, nil) } -func TestRFC5425Strict_unix_tls(t *testing.T) { +func TestOctetCountingStrict_unix_tls(t *testing.T) { tmpdir, err := ioutil.TempDir("", "telegraf") require.NoError(t, err) defer os.RemoveAll(tmpdir) sock := filepath.Join(tmpdir, "syslog.TestStrict_unix_tls.sock") - testStrictRFC5425(t, "unix", sock, true, nil) + testStrictOctetCounting(t, "unix", sock, true, nil) } -func TestRFC5425BestEffort_unix_tls(t *testing.T) { +func TestOctetCountingBestEffort_unix_tls(t *testing.T) { tmpdir, err := ioutil.TempDir("", "telegraf") require.NoError(t, err) defer os.RemoveAll(tmpdir) sock := filepath.Join(tmpdir, "syslog.TestBestEffort_unix_tls.sock") - testBestEffortRFC5425(t, "unix", sock, true, nil) + testBestEffortOctetCounting(t, "unix", sock, true, nil) } diff --git a/plugins/inputs/syslog/syslog.go b/plugins/inputs/syslog/syslog.go index b4cb2bff541f5..c7ff2dcc33d4b 100644 --- a/plugins/inputs/syslog/syslog.go +++ b/plugins/inputs/syslog/syslog.go @@ -13,9 +13,9 @@ import ( "unicode" "github.com/influxdata/go-syslog" + "github.com/influxdata/go-syslog/nontransparent" + "github.com/influxdata/go-syslog/octetcounting" "github.com/influxdata/go-syslog/rfc5424" - "github.com/influxdata/go-syslog/rfc5425" - "github.com/influxdata/go-syslog/rfc6587" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" tlsConfig "github.com/influxdata/telegraf/internal/tls" @@ -33,7 +33,7 @@ type Syslog struct { MaxConnections int ReadTimeout *internal.Duration TransparentFraming bool - Trailer rfc6587.TrailerType + Trailer nontransparent.TrailerType BestEffort bool Separator string `toml:"sdparam_separator"` @@ -82,7 +82,7 @@ var sampleConfig = ` ## Whether the messages come using the transparent framing or not (default = false). ## When false messages come using non-transparent framing technique (RFC6587#section-3.4.2). - ## True means messages come using octect-counting framing technique (RFC5425#section-4.3.1). + ## True means messages come using octect-counting framing technique (RFC5425#section-4.3.1 and RFC6587#section-3.4.1). # transparent_framing = false ## The trailer to be expected in case of non-trasparent framing (default = "LF"). @@ -313,12 +313,12 @@ func (s *Syslog) handle(conn net.Conn, acc telegraf.Accumulator) { // Select the parser to use depeding on transport framing if s.TransparentFraming { - // Octet-counting transparent framing - p = rfc5425.NewParser(opts...) + // Octet counting transparent framing + p = octetcounting.NewParser(opts...) } else { // Non-transparent framing - opts = append(opts, rfc6587.WithTrailer(s.Trailer)) - p = rfc6587.NewParser(opts...) + opts = append(opts, nontransparent.WithTrailer(s.Trailer)) + p = nontransparent.NewParser(opts...) } p.Parse(conn) @@ -443,7 +443,7 @@ func init() { ReadTimeout: &internal.Duration{ Duration: defaultReadTimeout, }, - Trailer: rfc6587.LF, + Trailer: nontransparent.LF, Separator: "_", } From 50af8bfc395464314668a8807be3e5f7ea26b0de Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Fri, 14 Dec 2018 16:54:54 +0100 Subject: [PATCH 07/11] Deps: Depend on go-syslog branch Signed-off-by: Leonardo Di Donato --- Gopkg.lock | 25 ++++++++++++++++++++----- Gopkg.toml | 2 +- plugins/inputs/syslog/README.md | 2 +- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index d043bccd005ad..40caf5482e74e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -583,15 +583,17 @@ version = "v0.8.1" [[projects]] - digest = "1:a39ef049cdeee03a57b132e7d60e32711b9d949c78458da78e702d9864c54369" + branch = "feature/rfc6587" + digest = "1:7db3a48fdc8b0ff3930cce77315e402535a2247b75cf86661b6a9d6f14d73823" name = "github.com/influxdata/go-syslog" packages = [ + ".", + "nontransparent", + "octetcounting", "rfc5424", - "rfc5425", ] pruneopts = "" - revision = "eecd51df3ad85464a2bab9b7d3a45bc1e299059e" - version = "v1.0.1" + revision = "5d2495909cc75526562d366079488d23cf1b2ae9" [[projects]] branch = "master" @@ -689,6 +691,17 @@ pruneopts = "" revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" +[[projects]] + branch = "develop" + digest = "1:3e66a61a57bbbe832c338edb3a623be0deb3dec650c2f3515149658898287e37" + name = "github.com/leodido/ragel-machinery" + packages = [ + ".", + "parser", + ] + pruneopts = "" + revision = "299bdde78165d4ca4bc7d064d8d6a4f39ac6de8c" + [[projects]] branch = "master" digest = "1:7e9956922e349af0190afa0b6621befcd201072679d8e51a9047ff149f2afe93" @@ -1478,8 +1491,10 @@ "github.com/google/go-cmp/cmp", "github.com/gorilla/mux", "github.com/hashicorp/consul/api", + "github.com/influxdata/go-syslog", + "github.com/influxdata/go-syslog/nontransparent", + "github.com/influxdata/go-syslog/octetcounting", "github.com/influxdata/go-syslog/rfc5424", - "github.com/influxdata/go-syslog/rfc5425", "github.com/influxdata/tail", "github.com/influxdata/toml", "github.com/influxdata/toml/ast", diff --git a/Gopkg.toml b/Gopkg.toml index 366ec6db9f7c1..cc77a569016a7 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -72,7 +72,7 @@ [[constraint]] name = "github.com/influxdata/go-syslog" - version = "feature/rfc6587" + branch = "feature/rfc6587" [[constraint]] name = "github.com/influxdata/tail" diff --git a/plugins/inputs/syslog/README.md b/plugins/inputs/syslog/README.md index 68018cda529a4..baa7a7a32ad32 100644 --- a/plugins/inputs/syslog/README.md +++ b/plugins/inputs/syslog/README.md @@ -3,7 +3,7 @@ The syslog plugin listens for syslog messages transmitted over [UDP](https://tools.ietf.org/html/rfc5426) or [TCP](https://tools.ietf.org/html/rfc6587) or -[TLS](https://tools.ietf.org/html/rfc5425), with or without the octet couting framing. +[TLS](https://tools.ietf.org/html/rfc5425), with or without the octet counting framing. Syslog messages should be formatted according to [RFC 5424](https://tools.ietf.org/html/rfc5424). From 453ca005d59b18a7dcf839578c2686b3bd5aa51c Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 17 Dec 2018 11:37:33 +0100 Subject: [PATCH 08/11] Docs: Update syslog readme --- plugins/inputs/syslog/README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/plugins/inputs/syslog/README.md b/plugins/inputs/syslog/README.md index baa7a7a32ad32..8183d2c9046c9 100644 --- a/plugins/inputs/syslog/README.md +++ b/plugins/inputs/syslog/README.md @@ -38,10 +38,11 @@ Syslog messages should be formatted according to ## 0 means unlimited. # read_timeout = "5s" - ## Whether the messages come using the transparent framing or not (default = false). - ## When false messages come using non-transparent framing technique (RFC6587#section-3.4.2). - ## True means messages come using octect-counting framing technique (RFC5425#section-4.3.1 and RFC6587#section-3.4.1). - # transparent_framing = false + ## The framing technique with which it is expected that messages are transported (default = "octet-counting"). + ## Whether the messages come using the octect-counting (RFC5425#section-4.3.1, RFC6587#section-3.4.1), + ## or the non-transparent framing technique (RFC6587#section-3.4.2). + ## Must be one of "octect-counting", "non-transparent". + # framing = "octet-counting" ## The trailer to be expected in case of non-trasparent framing (default = "LF"). ## Must be one of "LF", or "NUL". @@ -61,10 +62,10 @@ Syslog messages should be formatted according to #### Message transport -The `transparent_framing` option only applies to streams. It governs the way we expect to receive messages within the stream. -With the [octet counting](https://tools.ietf.org/html/rfc5425#section-4.3) technique or with the [non-transparent](https://tools.ietf.org/html/rfc6587#section-3.4.2) framing. +The `framing` option only applies to streams. It governs the way we expect to receive messages within the stream. +Namely, with the [`"octet counting"`](https://tools.ietf.org/html/rfc5425#section-4.3) technique (default) or with the [`"non-transparent"`](https://tools.ietf.org/html/rfc6587#section-3.4.2) framing. -The `trailer` option only applies when `transparent_framing` is `false` - ie., non-transparent transport. +The `trailer` option only applies when `framing` option is `"non-transparent"`. It must have one of the following values: `"LF"` (default), or `"NUL"`. #### Best effort From 074530ad89780b43c22994b7c05dbd6e37260c3b Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Mon, 17 Dec 2018 11:37:58 +0100 Subject: [PATCH 09/11] Update: Option framing as enum --- plugins/inputs/syslog/commons_test.go | 10 ++--- plugins/inputs/syslog/framing.go | 46 ++++++++++++++++++++ plugins/inputs/syslog/nontransparent_test.go | 4 +- plugins/inputs/syslog/octetcounting_test.go | 4 +- plugins/inputs/syslog/syslog.go | 28 ++++++------ 5 files changed, 70 insertions(+), 22 deletions(-) create mode 100644 plugins/inputs/syslog/framing.go diff --git a/plugins/inputs/syslog/commons_test.go b/plugins/inputs/syslog/commons_test.go index c21b742a3072a..0970f0a0b4045 100644 --- a/plugins/inputs/syslog/commons_test.go +++ b/plugins/inputs/syslog/commons_test.go @@ -37,7 +37,7 @@ func newUDPSyslogReceiver(address string, bestEffort bool) *Syslog { } } -func newTCPSyslogReceiver(address string, keepAlive *internal.Duration, maxConn int, bestEffort bool, transparent bool) *Syslog { +func newTCPSyslogReceiver(address string, keepAlive *internal.Duration, maxConn int, bestEffort bool, framing framing) *Syslog { d := &internal.Duration{ Duration: defaultReadTimeout, } @@ -46,10 +46,10 @@ func newTCPSyslogReceiver(address string, keepAlive *internal.Duration, maxConn now: func() time.Time { return defaultTime }, - TransparentFraming: transparent, - ReadTimeout: d, - BestEffort: bestEffort, - Separator: "_", + Framing: framing, + ReadTimeout: d, + BestEffort: bestEffort, + Separator: "_", } if keepAlive != nil { s.KeepAlivePeriod = keepAlive diff --git a/plugins/inputs/syslog/framing.go b/plugins/inputs/syslog/framing.go new file mode 100644 index 0000000000000..5bfb9eb8a1204 --- /dev/null +++ b/plugins/inputs/syslog/framing.go @@ -0,0 +1,46 @@ +package syslog + +import ( + "fmt" + "strings" +) + +type framing int + +const ( + // OctetCounting indicates the transparent framing technique for syslog transport. + OctetCounting framing = iota + // NonTransparent indicates the non-transparent framing technique for syslog transport. + NonTransparent +) + +func (f framing) String() string { + switch f { + case OctetCounting: + return "OCTET-COUNTING" + case NonTransparent: + return "NON-TRANSPARENT" + } + return "" +} + +// UnmarshalText implements encoding.TextUnmarshaler +func (f *framing) UnmarshalText(data []byte) (err error) { + s := string(data) + switch strings.ToUpper(s) { + case "OCTET-COUNTING": + *f = OctetCounting + case "NON-TRANSPARENT": + *f = NonTransparent + } + return err +} + +// MarshalText implements encoding.TextMarshaler +func (f framing) MarshalText() ([]byte, error) { + s := f.String() + if s != "" { + return []byte(s), nil + } + return nil, fmt.Errorf("unknown framing") +} diff --git a/plugins/inputs/syslog/nontransparent_test.go b/plugins/inputs/syslog/nontransparent_test.go index b3dbbc4e0e48a..1dea841448d7c 100644 --- a/plugins/inputs/syslog/nontransparent_test.go +++ b/plugins/inputs/syslog/nontransparent_test.go @@ -138,7 +138,7 @@ func testStrictNonTransparent(t *testing.T, protocol string, address string, wan for _, tc := range getTestCasesForNonTransparent() { t.Run(tc.name, func(t *testing.T) { // Creation of a strict mode receiver - receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, false, false) + receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, false, NonTransparent) require.NotNil(t, receiver) if wantTLS { receiver.ServerConfig = *pki.TLSServerConfig() @@ -200,7 +200,7 @@ func testBestEffortNonTransparent(t *testing.T, protocol string, address string, for _, tc := range getTestCasesForNonTransparent() { t.Run(tc.name, func(t *testing.T) { // Creation of a best effort mode receiver - receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, true, false) + receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, true, NonTransparent) require.NotNil(t, receiver) if wantTLS { receiver.ServerConfig = *pki.TLSServerConfig() diff --git a/plugins/inputs/syslog/octetcounting_test.go b/plugins/inputs/syslog/octetcounting_test.go index 77f416b6bd9e2..c61805131f594 100644 --- a/plugins/inputs/syslog/octetcounting_test.go +++ b/plugins/inputs/syslog/octetcounting_test.go @@ -338,7 +338,7 @@ func testStrictOctetCounting(t *testing.T, protocol string, address string, want for _, tc := range getTestCasesForOctetCounting() { t.Run(tc.name, func(t *testing.T) { // Creation of a strict mode receiver - receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, false, true) + receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, false, OctetCounting) require.NotNil(t, receiver) if wantTLS { receiver.ServerConfig = *pki.TLSServerConfig() @@ -400,7 +400,7 @@ func testBestEffortOctetCounting(t *testing.T, protocol string, address string, for _, tc := range getTestCasesForOctetCounting() { t.Run(tc.name, func(t *testing.T) { // Creation of a best effort mode receiver - receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, true, true) + receiver := newTCPSyslogReceiver(protocol+"://"+address, keepAlive, 0, true, OctetCounting) require.NotNil(t, receiver) if wantTLS { receiver.ServerConfig = *pki.TLSServerConfig() diff --git a/plugins/inputs/syslog/syslog.go b/plugins/inputs/syslog/syslog.go index c7ff2dcc33d4b..fa6b684ff3cd8 100644 --- a/plugins/inputs/syslog/syslog.go +++ b/plugins/inputs/syslog/syslog.go @@ -28,14 +28,14 @@ const ipMaxPacketSize = 64 * 1024 // Syslog is a syslog plugin type Syslog struct { tlsConfig.ServerConfig - Address string `toml:"server"` - KeepAlivePeriod *internal.Duration - MaxConnections int - ReadTimeout *internal.Duration - TransparentFraming bool - Trailer nontransparent.TrailerType - BestEffort bool - Separator string `toml:"sdparam_separator"` + Address string `toml:"server"` + KeepAlivePeriod *internal.Duration + MaxConnections int + ReadTimeout *internal.Duration + Framing framing + Trailer nontransparent.TrailerType + BestEffort bool + Separator string `toml:"sdparam_separator"` now func() time.Time lastTime time.Time @@ -80,10 +80,11 @@ var sampleConfig = ` ## 0 means unlimited. # read_timeout = "5s" - ## Whether the messages come using the transparent framing or not (default = false). - ## When false messages come using non-transparent framing technique (RFC6587#section-3.4.2). - ## True means messages come using octect-counting framing technique (RFC5425#section-4.3.1 and RFC6587#section-3.4.1). - # transparent_framing = false + ## The framing technique with which it is expected that messages are transported (default = "octet-counting"). + ## Whether the messages come using the octect-counting (RFC5425#section-4.3.1, RFC6587#section-3.4.1), + ## or the non-transparent framing technique (RFC6587#section-3.4.2). + ## Must be one of "octect-counting", "non-transparent". + # framing = "octet-counting" ## The trailer to be expected in case of non-trasparent framing (default = "LF"). ## Must be one of "LF", or "NUL". @@ -312,7 +313,7 @@ func (s *Syslog) handle(conn net.Conn, acc telegraf.Accumulator) { } // Select the parser to use depeding on transport framing - if s.TransparentFraming { + if s.Framing == OctetCounting { // Octet counting transparent framing p = octetcounting.NewParser(opts...) } else { @@ -443,6 +444,7 @@ func init() { ReadTimeout: &internal.Duration{ Duration: defaultReadTimeout, }, + Framing: OctetCounting, Trailer: nontransparent.LF, Separator: "_", } From 596b368eb3c464434f7c94934fd8298c5872ab87 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 18 Dec 2018 00:57:11 +0100 Subject: [PATCH 10/11] Fix: Decoding framing from TOML Signed-off-by: Leonardo Di Donato --- Gopkg.lock | 4 +-- plugins/inputs/syslog/commons_test.go | 4 +-- plugins/inputs/syslog/framing.go | 34 ++++++++++++++++++------ plugins/inputs/syslog/framing_test.go | 37 +++++++++++++++++++++++++++ plugins/inputs/syslog/syslog.go | 2 +- 5 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 plugins/inputs/syslog/framing_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 40caf5482e74e..8cd346e075276 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -584,7 +584,7 @@ [[projects]] branch = "feature/rfc6587" - digest = "1:7db3a48fdc8b0ff3930cce77315e402535a2247b75cf86661b6a9d6f14d73823" + digest = "1:824c4cd143ee15735f1c75d9072aad46e51dd27a4ef8bf6ce723a138265b7fb3" name = "github.com/influxdata/go-syslog" packages = [ ".", @@ -593,7 +593,7 @@ "rfc5424", ] pruneopts = "" - revision = "5d2495909cc75526562d366079488d23cf1b2ae9" + revision = "55af43430773a14e1614702e67f39f9ca4867b60" [[projects]] branch = "master" diff --git a/plugins/inputs/syslog/commons_test.go b/plugins/inputs/syslog/commons_test.go index 0970f0a0b4045..f55d080a13fba 100644 --- a/plugins/inputs/syslog/commons_test.go +++ b/plugins/inputs/syslog/commons_test.go @@ -37,7 +37,7 @@ func newUDPSyslogReceiver(address string, bestEffort bool) *Syslog { } } -func newTCPSyslogReceiver(address string, keepAlive *internal.Duration, maxConn int, bestEffort bool, framing framing) *Syslog { +func newTCPSyslogReceiver(address string, keepAlive *internal.Duration, maxConn int, bestEffort bool, f Framing) *Syslog { d := &internal.Duration{ Duration: defaultReadTimeout, } @@ -46,7 +46,7 @@ func newTCPSyslogReceiver(address string, keepAlive *internal.Duration, maxConn now: func() time.Time { return defaultTime }, - Framing: framing, + Framing: f, ReadTimeout: d, BestEffort: bestEffort, Separator: "_", diff --git a/plugins/inputs/syslog/framing.go b/plugins/inputs/syslog/framing.go index 5bfb9eb8a1204..6edfc70587d0d 100644 --- a/plugins/inputs/syslog/framing.go +++ b/plugins/inputs/syslog/framing.go @@ -5,16 +5,17 @@ import ( "strings" ) -type framing int +// Framing represents the framing technique we expect the messages to come. +type Framing int const ( // OctetCounting indicates the transparent framing technique for syslog transport. - OctetCounting framing = iota + OctetCounting Framing = iota // NonTransparent indicates the non-transparent framing technique for syslog transport. NonTransparent ) -func (f framing) String() string { +func (f Framing) String() string { switch f { case OctetCounting: return "OCTET-COUNTING" @@ -24,20 +25,37 @@ func (f framing) String() string { return "" } +// UnmarshalTOML implements ability to unmarshal framing from TOML files. +func (f *Framing) UnmarshalTOML(data []byte) (err error) { + return f.UnmarshalText(data) +} + // UnmarshalText implements encoding.TextUnmarshaler -func (f *framing) UnmarshalText(data []byte) (err error) { +func (f *Framing) UnmarshalText(data []byte) (err error) { s := string(data) switch strings.ToUpper(s) { - case "OCTET-COUNTING": + case `OCTET-COUNTING`: + fallthrough + case `"OCTET-COUNTING"`: + fallthrough + case `'OCTET-COUNTING'`: *f = OctetCounting - case "NON-TRANSPARENT": + return + + case `NON-TRANSPARENT`: + fallthrough + case `"NON-TRANSPARENT"`: + fallthrough + case `'NON-TRANSPARENT'`: *f = NonTransparent + return } - return err + *f = -1 + return fmt.Errorf("unknown framing") } // MarshalText implements encoding.TextMarshaler -func (f framing) MarshalText() ([]byte, error) { +func (f Framing) MarshalText() ([]byte, error) { s := f.String() if s != "" { return []byte(s), nil diff --git a/plugins/inputs/syslog/framing_test.go b/plugins/inputs/syslog/framing_test.go new file mode 100644 index 0000000000000..1442eba7f630f --- /dev/null +++ b/plugins/inputs/syslog/framing_test.go @@ -0,0 +1,37 @@ +package syslog + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestFraming(t *testing.T) { + var f1 Framing + f1.UnmarshalTOML([]byte(`"non-transparent"`)) + assert.Equal(t, NonTransparent, f1) + + var f2 Framing + f2.UnmarshalTOML([]byte(`non-transparent`)) + assert.Equal(t, NonTransparent, f2) + + var f3 Framing + f3.UnmarshalTOML([]byte(`'non-transparent'`)) + assert.Equal(t, NonTransparent, f3) + + var f4 Framing + f4.UnmarshalTOML([]byte(`"octet-counting"`)) + assert.Equal(t, OctetCounting, f4) + + var f5 Framing + f5.UnmarshalTOML([]byte(`octet-counting`)) + assert.Equal(t, OctetCounting, f5) + + var f6 Framing + f6.UnmarshalTOML([]byte(`'octet-counting'`)) + assert.Equal(t, OctetCounting, f6) + + var f7 Framing + err := f7.UnmarshalTOML([]byte(`nope`)) + assert.Equal(t, Framing(-1), f7) + assert.Error(t, err) +} diff --git a/plugins/inputs/syslog/syslog.go b/plugins/inputs/syslog/syslog.go index fa6b684ff3cd8..ab2277caa69fe 100644 --- a/plugins/inputs/syslog/syslog.go +++ b/plugins/inputs/syslog/syslog.go @@ -32,7 +32,7 @@ type Syslog struct { KeepAlivePeriod *internal.Duration MaxConnections int ReadTimeout *internal.Duration - Framing framing + Framing Framing Trailer nontransparent.TrailerType BestEffort bool Separator string `toml:"sdparam_separator"` From dc70733cd6b42e915a5d54e3ebfd0344908264e5 Mon Sep 17 00:00:00 2001 From: Leonardo Di Donato Date: Tue, 18 Dec 2018 11:23:15 +0100 Subject: [PATCH 11/11] Deps: Pin to go-syslog 2.0.0 version Signed-off-by: Leonardo Di Donato --- Gopkg.lock | 4 ++-- Gopkg.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 8cd346e075276..8fd3e81c499b0 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -583,7 +583,6 @@ version = "v0.8.1" [[projects]] - branch = "feature/rfc6587" digest = "1:824c4cd143ee15735f1c75d9072aad46e51dd27a4ef8bf6ce723a138265b7fb3" name = "github.com/influxdata/go-syslog" packages = [ @@ -593,7 +592,8 @@ "rfc5424", ] pruneopts = "" - revision = "55af43430773a14e1614702e67f39f9ca4867b60" + revision = "0cd00a9f0a5e5607d5ef9a294c260f77a74e3b5a" + version = "v2.0.0" [[projects]] branch = "master" diff --git a/Gopkg.toml b/Gopkg.toml index cc77a569016a7..3b5c1b917472f 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -72,7 +72,7 @@ [[constraint]] name = "github.com/influxdata/go-syslog" - branch = "feature/rfc6587" + version = "2.0.0" [[constraint]] name = "github.com/influxdata/tail"