From 1b6c1bdf8cc825371b74e76ee338f670a3a58571 Mon Sep 17 00:00:00 2001 From: Corbin Phelps Date: Mon, 24 Oct 2022 16:14:01 -0400 Subject: [PATCH] [syslogreceiver] Add for RFC 6587 octet counting and non-transparent-framing (#15358) * Added support for syslog rfc 6587 octet counting and non-transparent-framing Signed-off-by: Corbin Phelps --- .chloggen/syslogreceiver-rfc6587-support.yaml | 16 +++ cmd/configschema/go.mod | 1 + cmd/configschema/go.sum | 1 + go.mod | 1 + go.sum | 1 + pkg/stanza/docs/operators/syslog_parser.md | 26 ++--- pkg/stanza/go.mod | 1 + pkg/stanza/go.sum | 1 + pkg/stanza/operator/input/syslog/syslog.go | 7 ++ .../operator/input/syslog/syslog_test.go | 17 ++-- .../operator/parser/syslog/config_test.go | 92 +++++++++++++++++ pkg/stanza/operator/parser/syslog/data.go | 97 ++++++++++++++++++ pkg/stanza/operator/parser/syslog/syslog.go | 99 ++++++++++++++++--- receiver/syslogreceiver/README.md | 2 + receiver/syslogreceiver/go.mod | 1 + receiver/syslogreceiver/go.sum | 1 + 16 files changed, 331 insertions(+), 33 deletions(-) create mode 100644 .chloggen/syslogreceiver-rfc6587-support.yaml diff --git a/.chloggen/syslogreceiver-rfc6587-support.yaml b/.chloggen/syslogreceiver-rfc6587-support.yaml new file mode 100644 index 000000000000..4e36e6bec5ae --- /dev/null +++ b/.chloggen/syslogreceiver-rfc6587-support.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: "enhancement" + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: syslogreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Added RFC 6587 Octet Counting and Non-Transparent-Framing support. + +# One or more tracking issues related to the change +issues: [8390] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/cmd/configschema/go.mod b/cmd/configschema/go.mod index 3cff553d0bab..795c81ca9c1f 100644 --- a/cmd/configschema/go.mod +++ b/cmd/configschema/go.mod @@ -273,6 +273,7 @@ require ( github.com/knadh/koanf v1.4.3 // indirect github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 // indirect github.com/leoluk/perflib_exporter v0.2.0 // indirect github.com/lib/pq v1.10.7 // indirect github.com/lightstep/go-expohisto v1.0.0 // indirect diff --git a/cmd/configschema/go.sum b/cmd/configschema/go.sum index 4109f4fb7cce..7f974cbe6ba9 100644 --- a/cmd/configschema/go.sum +++ b/cmd/configschema/go.sum @@ -1372,6 +1372,7 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg= github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 h1:X/79QL0b4YJVO5+OsPH9rF2u428CIrGL/jLmPsoOQQ4= +github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 h1:bCiVCRCs1Heq84lurVinUPy19keqGEe4jh5vtK37jcg= github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= github.com/leoluk/perflib_exporter v0.2.0 h1:WJU7N3AIHxfc3CjoEJcBgG3i2ltF5Yz1ADVY9T6f1BY= github.com/leoluk/perflib_exporter v0.2.0/go.mod h1:MinSWm88jguXFFrGsP56PtleUb4Qtm4tNRH/wXNXRTI= diff --git a/go.mod b/go.mod index 9f7c4d4639b6..9593388985f5 100644 --- a/go.mod +++ b/go.mod @@ -425,6 +425,7 @@ require ( github.com/knadh/koanf v1.4.3 // indirect github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 // indirect github.com/leoluk/perflib_exporter v0.2.0 // indirect github.com/lib/pq v1.10.7 // indirect github.com/lightstep/go-expohisto v1.0.0 // indirect diff --git a/go.sum b/go.sum index 9a932cf2c41f..fc021d1a140d 100644 --- a/go.sum +++ b/go.sum @@ -1369,6 +1369,7 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg= github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 h1:X/79QL0b4YJVO5+OsPH9rF2u428CIrGL/jLmPsoOQQ4= +github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 h1:bCiVCRCs1Heq84lurVinUPy19keqGEe4jh5vtK37jcg= github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= github.com/leoluk/perflib_exporter v0.2.0 h1:WJU7N3AIHxfc3CjoEJcBgG3i2ltF5Yz1ADVY9T6f1BY= github.com/leoluk/perflib_exporter v0.2.0/go.mod h1:MinSWm88jguXFFrGsP56PtleUb4Qtm4tNRH/wXNXRTI= diff --git a/pkg/stanza/docs/operators/syslog_parser.md b/pkg/stanza/docs/operators/syslog_parser.md index 409ec354598f..4606c5fa3d45 100644 --- a/pkg/stanza/docs/operators/syslog_parser.md +++ b/pkg/stanza/docs/operators/syslog_parser.md @@ -4,18 +4,20 @@ The `syslog_parser` operator parses the string-type field selected by `parse_fro ### Configuration Fields -| Field | Default | Description | -| --- | --- | --- | -| `id` | `syslog_parser` | A unique identifier for the operator. | -| `output` | Next in pipeline | The connected operator(s) that will receive all outbound entries. | -| `parse_from` | `body` | The [field](../types/field.md) from which the value will be parsed. | -| `parse_to` | `attributes` | The [field](../types/field.md) to which the value will be parsed. | -| `on_error` | `send` | The behavior of the operator if it encounters an error. See [on_error](../types/on_error.md). | -| `protocol` | required | The protocol to parse the syslog messages as. Options are `rfc3164` and `rfc5424`. | -| `location` | `UTC` | The geographic location (timezone) to use when parsing the timestamp (Syslog RFC 3164 only). The available locations depend on the local IANA Time Zone database. [This page](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) contains many examples, such as `America/New_York`. | -| `timestamp` | `nil` | An optional [timestamp](../types/timestamp.md) block which will parse a timestamp field before passing the entry to the output operator | -| `severity` | `nil` | An optional [severity](../types/severity.md) block which will parse a severity field before passing the entry to the output operator | -| `if` | | An [expression](../types/expression.md) that, when set, will be evaluated to determine whether this operator should be used for the given entry. This allows you to do easy conditional parsing without branching logic with routers. | +| Field | Default | Description | +| --- | --- | --- | +| `id` | `syslog_parser` | A unique identifier for the operator. | +| `output` | Next in pipeline | The connected operator(s) that will receive all outbound entries. | +| `parse_from` | `body` | The [field](../types/field.md) from which the value will be parsed. | +| `parse_to` | `attributes` | The [field](../types/field.md) to which the value will be parsed. | +| `on_error` | `send` | The behavior of the operator if it encounters an error. See [on_error](../types/on_error.md). | +| `protocol` | required | The protocol to parse the syslog messages as. Options are `rfc3164` and `rfc5424`. | +| `location` | `UTC` | The geographic location (timezone) to use when parsing the timestamp (Syslog RFC 3164 only). The available locations depend on the local IANA Time Zone database. [This page](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) contains many examples, such as `America/New_York`. | +| `enable_octet_counting` | `false` | Wether or not to enable [RFC 6587](https://www.rfc-editor.org/rfc/rfc6587#section-3.4.1) Octet Counting on syslog parsing (Syslog RFC 5424 only). | +| `non_transparent_framing_trailer` | `nil` | The framing trailer, either `LF` or `NUL`, when using [RFC 6587](https://www.rfc-editor.org/rfc/rfc6587#section-3.4.2) Non-Transparent-Framing (Syslog RFC 5424 only). | +| `timestamp` | `nil` | An optional [timestamp](../types/timestamp.md) block which will parse a timestamp field before passing the entry to the output operator | +| `severity` | `nil` | An optional [severity](../types/severity.md) block which will parse a severity field before passing the entry to the output operator | +| `if` | | An [expression](../types/expression.md) that, when set, will be evaluated to determine whether this operator should be used for the given entry. This allows you to do easy conditional parsing without branching logic with routers. | ### Embedded Operations diff --git a/pkg/stanza/go.mod b/pkg/stanza/go.mod index cce1e1ebe300..e7beecc4c829 100644 --- a/pkg/stanza/go.mod +++ b/pkg/stanza/go.mod @@ -36,6 +36,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/knadh/koanf v1.4.3 // indirect + github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/pkg/stanza/go.sum b/pkg/stanza/go.sum index fbb3638902ce..d0f65f1475ec 100644 --- a/pkg/stanza/go.sum +++ b/pkg/stanza/go.sum @@ -184,6 +184,7 @@ github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 h1:bCiVCRCs1Heq84lurVinUPy19keqGEe4jh5vtK37jcg= github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= diff --git a/pkg/stanza/operator/input/syslog/syslog.go b/pkg/stanza/operator/input/syslog/syslog.go index b39e0850e947..2b62d1811a89 100644 --- a/pkg/stanza/operator/input/syslog/syslog.go +++ b/pkg/stanza/operator/input/syslog/syslog.go @@ -15,6 +15,7 @@ package syslog // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/input/syslog" import ( + "errors" "fmt" "go.uber.org/zap" @@ -88,9 +89,15 @@ func (c Config) Build(logger *zap.SugaredLogger) (operator.Operator, error) { } if c.UDP != nil { + udpInputCfg := udp.NewConfigWithID(inputBase.ID() + "_internal_udp") udpInputCfg.BaseConfig = *c.UDP + // Octet counting and Non-Transparent-Framing are invalid for UDP connections + if syslogParserCfg.EnableOctetCounting || syslogParserCfg.NonTransparentFramingTrailer != nil { + return nil, errors.New("octet_counting and non_transparent_framing is not compatible with UDP") + } + udpInput, err := udpInputCfg.Build(logger) if err != nil { return nil, fmt.Errorf("failed to resolve upd config: %w", err) diff --git a/pkg/stanza/operator/input/syslog/syslog_test.go b/pkg/stanza/operator/input/syslog/syslog_test.go index 7f8a9b9d1305..5956d5327a50 100644 --- a/pkg/stanza/operator/input/syslog/syslog_test.go +++ b/pkg/stanza/operator/input/syslog/syslog_test.go @@ -40,12 +40,17 @@ func TestInput(t *testing.T) { require.NoError(t, err) for _, tc := range cases { - t.Run(fmt.Sprintf("TCP-%s", tc.Name), func(t *testing.T) { - InputTest(t, NewConfigWithTCP(&tc.Config.BaseConfig), tc) - }) - t.Run(fmt.Sprintf("UDP-%s", tc.Name), func(t *testing.T) { - InputTest(t, NewConfigWithUDP(&tc.Config.BaseConfig), tc) - }) + if tc.ValidForTCP { + t.Run(fmt.Sprintf("TCP-%s", tc.Name), func(t *testing.T) { + InputTest(t, NewConfigWithTCP(&tc.Config.BaseConfig), tc) + }) + } + + if tc.ValidForUDP { + t.Run(fmt.Sprintf("UDP-%s", tc.Name), func(t *testing.T) { + InputTest(t, NewConfigWithUDP(&tc.Config.BaseConfig), tc) + }) + } } } diff --git a/pkg/stanza/operator/parser/syslog/config_test.go b/pkg/stanza/operator/parser/syslog/config_test.go index 423e999aa900..93eef08d4933 100644 --- a/pkg/stanza/operator/parser/syslog/config_test.go +++ b/pkg/stanza/operator/parser/syslog/config_test.go @@ -142,6 +142,98 @@ func TestParserMissingProtocol(t *testing.T) { require.Contains(t, err.Error(), "missing field 'protocol'") } +func TestRFC6587ConfigOptions(t *testing.T) { + validFramingTrailer := NULTrailer + invalidFramingTrailer := "bad" + testCases := []struct { + desc string + cfg *Config + errContents string + }{ + { + desc: "Octet Counting with RFC3164", + cfg: &Config{ + ParserConfig: helper.NewParserConfig(operatorType, operatorType), + BaseConfig: BaseConfig{ + Protocol: RFC3164, + EnableOctetCounting: true, + }, + }, + errContents: "octet_counting and non_transparent_framing are only compatible with protocol rfc5424", + }, + { + desc: "Non-Transparent-Framing with RFC3164", + cfg: &Config{ + ParserConfig: helper.NewParserConfig(operatorType, operatorType), + BaseConfig: BaseConfig{ + Protocol: RFC3164, + NonTransparentFramingTrailer: &validFramingTrailer, + }, + }, + errContents: "octet_counting and non_transparent_framing are only compatible with protocol rfc5424", + }, + { + desc: "Non-Transparent-Framing and Octet counting both enabled with RFC5424", + cfg: &Config{ + ParserConfig: helper.NewParserConfig(operatorType, operatorType), + BaseConfig: BaseConfig{ + Protocol: RFC5424, + NonTransparentFramingTrailer: &validFramingTrailer, + EnableOctetCounting: true, + }, + }, + errContents: "only one of octet_counting or non_transparent_framing can be enabled", + }, + { + desc: "Valid Octet Counting", + cfg: &Config{ + ParserConfig: helper.NewParserConfig(operatorType, operatorType), + BaseConfig: BaseConfig{ + Protocol: RFC5424, + NonTransparentFramingTrailer: nil, + EnableOctetCounting: true, + }, + }, + errContents: "", + }, + { + desc: "Valid Non-Transparent-Framing Trailer", + cfg: &Config{ + ParserConfig: helper.NewParserConfig(operatorType, operatorType), + BaseConfig: BaseConfig{ + Protocol: RFC5424, + NonTransparentFramingTrailer: &validFramingTrailer, + EnableOctetCounting: false, + }, + }, + errContents: "", + }, + { + desc: "Invalid Non-Transparent-Framing Trailer", + cfg: &Config{ + ParserConfig: helper.NewParserConfig(operatorType, operatorType), + BaseConfig: BaseConfig{ + Protocol: RFC5424, + NonTransparentFramingTrailer: &invalidFramingTrailer, + EnableOctetCounting: false, + }, + }, + errContents: "invalid non_transparent_framing_trailer", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + _, err := tc.cfg.Build(testutil.Logger(t)) + if tc.errContents != "" { + require.ErrorContains(t, err, tc.errContents) + } else { + require.NoError(t, err) + } + }) + } +} + func TestParserInvalidLocation(t *testing.T) { config := NewConfig() config.Location = "not_a_location" diff --git a/pkg/stanza/operator/parser/syslog/data.go b/pkg/stanza/operator/parser/syslog/data.go index 26ce389b67cd..ca7e153a753c 100644 --- a/pkg/stanza/operator/parser/syslog/data.go +++ b/pkg/stanza/operator/parser/syslog/data.go @@ -15,6 +15,7 @@ package syslog // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/parser/syslog" import ( + "strings" "time" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/entry" @@ -25,6 +26,10 @@ type Case struct { Config *Config Input *entry.Entry Expect *entry.Entry + + // These signal if a test is valid for UDP and/or TCP protocol + ValidForTCP bool + ValidForUDP bool } func testLocations() (map[string]*time.Location, error) { @@ -50,6 +55,14 @@ func CreateCases(basicConfig func() *Config) ([]Case, error) { return nil, err } + // We need to build out the Non-Transparent-Framing body to ensure we control the Trailer byte + nonTransparentBodyBuilder := strings.Builder{} + nonTransparentBodyBuilder.WriteString(`<86>1 2015-08-05T21:58:59.693Z 192.168.2.132 SecureAuth0 23108 ID52020 [SecureAuth@27389 UserHostAddress="192.168.2.132" Realm="SecureAuth0" UserID="Tester2" PEN="27389"] Found the user for retrieving user's profile`) + nonTransparentBodyBuilder.WriteByte(0x00) + nonTransparentBody := nonTransparentBodyBuilder.String() + + nulFramingTrailer := NULTrailer + var cases = []Case{ { "RFC3164", @@ -75,6 +88,8 @@ func CreateCases(basicConfig func() *Config) ([]Case, error) { }, Body: "<34>Jan 12 06:30:00 1.2.3.4 apache_server: test message", }, + true, + true, }, { "RFC3164Detroit", @@ -100,6 +115,8 @@ func CreateCases(basicConfig func() *Config) ([]Case, error) { }, Body: "<34>Jan 12 06:30:00 1.2.3.4 apache_server: test message", }, + true, + true, }, { "RFC3164Athens", @@ -125,6 +142,8 @@ func CreateCases(basicConfig func() *Config) ([]Case, error) { }, Body: "<34>Jan 12 06:30:00 1.2.3.4 apache_server: test message", }, + true, + true, }, { "RFC5424", @@ -160,6 +179,84 @@ func CreateCases(basicConfig func() *Config) ([]Case, error) { }, Body: `<86>1 2015-08-05T21:58:59.693Z 192.168.2.132 SecureAuth0 23108 ID52020 [SecureAuth@27389 UserHostAddress="192.168.2.132" Realm="SecureAuth0" UserID="Tester2" PEN="27389"] Found the user for retrieving user's profile`, }, + true, + true, + }, + { + "RFC6587 Octet Counting", + func() *Config { + cfg := basicConfig() + cfg.Protocol = RFC5424 + cfg.EnableOctetCounting = true + return cfg + }(), + &entry.Entry{ + Body: `215 <86>1 2015-08-05T21:58:59.693Z 192.168.2.132 SecureAuth0 23108 ID52020 [SecureAuth@27389 UserHostAddress="192.168.2.132" Realm="SecureAuth0" UserID="Tester2" PEN="27389"] Found the user for retrieving user's profile`, + }, + &entry.Entry{ + Timestamp: time.Date(2015, 8, 5, 21, 58, 59, 693000000, time.UTC), + Severity: entry.Info, + SeverityText: "info", + Attributes: map[string]interface{}{ + "appname": "SecureAuth0", + "facility": 10, + "hostname": "192.168.2.132", + "message": "Found the user for retrieving user's profile", + "msg_id": "ID52020", + "priority": 86, + "proc_id": "23108", + "structured_data": map[string]map[string]string{ + "SecureAuth@27389": { + "PEN": "27389", + "Realm": "SecureAuth0", + "UserHostAddress": "192.168.2.132", + "UserID": "Tester2", + }, + }, + "version": 1, + }, + Body: `215 <86>1 2015-08-05T21:58:59.693Z 192.168.2.132 SecureAuth0 23108 ID52020 [SecureAuth@27389 UserHostAddress="192.168.2.132" Realm="SecureAuth0" UserID="Tester2" PEN="27389"] Found the user for retrieving user's profile`, + }, + true, + false, + }, + { + "RFC6587 Non-Transparent-framing", + func() *Config { + cfg := basicConfig() + cfg.Protocol = RFC5424 + cfg.NonTransparentFramingTrailer = &nulFramingTrailer + return cfg + }(), + &entry.Entry{ + Body: nonTransparentBody, + }, + &entry.Entry{ + Timestamp: time.Date(2015, 8, 5, 21, 58, 59, 693000000, time.UTC), + Severity: entry.Info, + SeverityText: "info", + Attributes: map[string]interface{}{ + "appname": "SecureAuth0", + "facility": 10, + "hostname": "192.168.2.132", + "message": "Found the user for retrieving user's profile", + "msg_id": "ID52020", + "priority": 86, + "proc_id": "23108", + "structured_data": map[string]map[string]string{ + "SecureAuth@27389": { + "PEN": "27389", + "Realm": "SecureAuth0", + "UserHostAddress": "192.168.2.132", + "UserID": "Tester2", + }, + }, + "version": 1, + }, + Body: nonTransparentBody, + }, + true, + false, }, } diff --git a/pkg/stanza/operator/parser/syslog/syslog.go b/pkg/stanza/operator/parser/syslog/syslog.go index feccd09243ef..43af6747b2e9 100644 --- a/pkg/stanza/operator/parser/syslog/syslog.go +++ b/pkg/stanza/operator/parser/syslog/syslog.go @@ -15,11 +15,15 @@ package syslog // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/parser/syslog" import ( + "bytes" "context" + "errors" "fmt" "time" sl "github.com/influxdata/go-syslog/v3" + "github.com/influxdata/go-syslog/v3/nontransparent" + "github.com/influxdata/go-syslog/v3/octetcounting" "github.com/influxdata/go-syslog/v3/rfc3164" "github.com/influxdata/go-syslog/v3/rfc5424" "go.uber.org/zap" @@ -34,6 +38,9 @@ const ( RFC3164 = "rfc3164" RFC5424 = "rfc5424" + + NULTrailer = "NUL" + LFTrailer = "LF" ) func init() { @@ -60,8 +67,10 @@ type Config struct { // BaseConfig is the detailed configuration of a syslog parser. type BaseConfig struct { - Protocol string `mapstructure:"protocol,omitempty"` - Location string `mapstructure:"location,omitempty"` + Protocol string `mapstructure:"protocol,omitempty"` + Location string `mapstructure:"location,omitempty"` + EnableOctetCounting bool `mapstructure:"enable_octet_counting,omitempty"` + NonTransparentFramingTrailer *string `mapstructure:"non_transparent_framing_trailer,omitempty"` } // Build will build a JSON parser operator. @@ -79,8 +88,17 @@ func (c Config) Build(logger *zap.SugaredLogger) (operator.Operator, error) { return nil, err } - if c.Protocol == "" { + switch { + case c.Protocol == "": return nil, fmt.Errorf("missing field 'protocol'") + case c.Protocol != RFC5424 && (c.NonTransparentFramingTrailer != nil || c.EnableOctetCounting): + return nil, errors.New("octet_counting and non_transparent_framing are only compatible with protocol rfc5424") + case c.Protocol == RFC5424 && (c.NonTransparentFramingTrailer != nil && c.EnableOctetCounting): + return nil, errors.New("only one of octet_counting or non_transparent_framing can be enabled") + case c.Protocol == RFC5424 && c.NonTransparentFramingTrailer != nil: + if *c.NonTransparentFramingTrailer != NULTrailer && *c.NonTransparentFramingTrailer != LFTrailer { + return nil, fmt.Errorf("invalid non_transparent_framing_trailer '%s'. Must be either 'LF' or 'NUL'", *c.NonTransparentFramingTrailer) + } } if c.Location == "" { @@ -93,28 +111,52 @@ func (c Config) Build(logger *zap.SugaredLogger) (operator.Operator, error) { } return &Parser{ - ParserOperator: parserOperator, - protocol: c.Protocol, - location: location, + ParserOperator: parserOperator, + protocol: c.Protocol, + location: location, + enableOctetCounting: c.EnableOctetCounting, + nonTransparentFramingTrailer: c.NonTransparentFramingTrailer, }, nil } -func buildMachine(protocol string, location *time.Location) (sl.Machine, error) { - switch protocol { +// parseFunc a parseFunc determines how the raw input is to be parsed into a syslog message +type parseFunc func(input []byte) (sl.Message, error) + +func (s *Parser) buildParseFunc() (parseFunc, error) { + switch s.protocol { case RFC3164: - return rfc3164.NewMachine(rfc3164.WithLocaleTimezone(location)), nil + return func(input []byte) (sl.Message, error) { + return rfc3164.NewMachine(rfc3164.WithLocaleTimezone(s.location)).Parse(input) + }, nil case RFC5424: - return rfc5424.NewMachine(), nil + switch { + // Octet Counting Parsing RFC6587 + case s.enableOctetCounting: + return newOctetCountingParseFunc(), nil + // Non-Transparent-Framing Parsing RFC6587 + case s.nonTransparentFramingTrailer != nil && *s.nonTransparentFramingTrailer == LFTrailer: + return newNonTransparentFramingParseFunc(nontransparent.LF), nil + case s.nonTransparentFramingTrailer != nil && *s.nonTransparentFramingTrailer == NULTrailer: + return newNonTransparentFramingParseFunc(nontransparent.NUL), nil + // Raw RFC5424 parsing + default: + return func(input []byte) (sl.Message, error) { + return rfc5424.NewMachine().Parse(input) + }, nil + } + default: - return nil, fmt.Errorf("invalid protocol %s", protocol) + return nil, fmt.Errorf("invalid protocol %s", s.protocol) } } // Parser is an operator that parses syslog. type Parser struct { helper.ParserOperator - protocol string - location *time.Location + protocol string + location *time.Location + enableOctetCounting bool + nonTransparentFramingTrailer *string } // Process will parse an entry field as syslog. @@ -129,12 +171,12 @@ func (s *Parser) parse(value interface{}) (interface{}, error) { return nil, err } - machine, err := buildMachine(s.protocol, s.location) + pFunc, err := s.buildParseFunc() if err != nil { return nil, err } - slog, err := machine.Parse(bytes) + slog, err := pFunc(bytes) if err != nil { return nil, err } @@ -279,3 +321,30 @@ func postprocess(e *entry.Entry) error { return nil } + +func newOctetCountingParseFunc() parseFunc { + return func(input []byte) (message sl.Message, err error) { + listener := func(res *sl.Result) { + message = res.Message + err = res.Error + } + parser := octetcounting.NewParser(sl.WithBestEffort(), sl.WithListener(listener)) + reader := bytes.NewReader(input) + parser.Parse(reader) + return + } +} + +func newNonTransparentFramingParseFunc(trailerType nontransparent.TrailerType) parseFunc { + return func(input []byte) (message sl.Message, err error) { + listener := func(res *sl.Result) { + message = res.Message + err = res.Error + } + + parser := nontransparent.NewParser(sl.WithBestEffort(), nontransparent.WithTrailer(trailerType), sl.WithListener(listener)) + reader := bytes.NewReader(input) + parser.Parse(reader) + return + } +} diff --git a/receiver/syslogreceiver/README.md b/receiver/syslogreceiver/README.md index 630aab2593b8..6848620c6f8b 100644 --- a/receiver/syslogreceiver/README.md +++ b/receiver/syslogreceiver/README.md @@ -16,6 +16,8 @@ Parses Syslogs received over TCP or UDP. | `udp` |`nil` | Defined udp_input operator. (see the UDP configuration section) | | `protocol` | required | The protocol to parse the syslog messages as. Options are `rfc3164` and `rfc5424` | | `location` | `UTC` | The geographic location (timezone) to use when parsing the timestamp (Syslog RFC 3164 only). The available locations depend on the local IANA Time Zone database. [This page](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) contains many examples, such as `America/New_York`. | +| `enable_octet_counting` | `false` | Wether or not to enable [RFC 6587](https://www.rfc-editor.org/rfc/rfc6587#section-3.4.1) Octet Counting on syslog parsing (Syslog RFC 5424 and TCP only). | +| `non_transparent_framing_trailer` | `nil` | The framing trailer, either `LF` or `NUL`, when using [RFC 6587](https://www.rfc-editor.org/rfc/rfc6587#section-3.4.2) Non-Transparent-Framing (Syslog RFC 5424 and TCP only). | | `timestamp` | `nil` | An optional [timestamp](../../pkg/stanza/docs/types/timestamp.md) block which will parse a timestamp field before passing the entry to the output operator | | `severity` | `nil` | An optional [severity](../../pkg/stanza/docs/types/severity.md) block which will parse a severity field before passing the entry to the output operator | `attributes` | {} | A map of `key: value` labels to add to the entry's attributes | diff --git a/receiver/syslogreceiver/go.mod b/receiver/syslogreceiver/go.mod index b94787944a31..2d82ef8a855e 100644 --- a/receiver/syslogreceiver/go.mod +++ b/receiver/syslogreceiver/go.mod @@ -22,6 +22,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/knadh/koanf v1.4.3 // indirect github.com/kr/pretty v0.3.0 // indirect + github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect diff --git a/receiver/syslogreceiver/go.sum b/receiver/syslogreceiver/go.sum index 972e66dae1a3..b374bdbbf5a0 100644 --- a/receiver/syslogreceiver/go.sum +++ b/receiver/syslogreceiver/go.sum @@ -183,6 +183,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 h1:bCiVCRCs1Heq84lurVinUPy19keqGEe4jh5vtK37jcg= github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=