Skip to content

Commit

Permalink
Merge pull request #2141 from okkez/improve-parser-syslog
Browse files Browse the repository at this point in the history 
Improve parser_syslog regexp for RFC5424
  • Loading branch information
@repeatedly
repeatedly authored Oct 2, 2018
2 parents 29ac632 + d9810b4 commit f4a6f09
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 3 deletions.
4 changes: 2 additions & 2 deletions lib/fluent/plugin/parser_syslog.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ class SyslogParser < Parser
REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
# From in_syslog default pattern
REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>.{1,128}) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>.{1,128}) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/

config_set_default :time_format, "%b %d %H:%M:%S"
Expand Down
85 changes: 84 additions & 1 deletion test/plugin/test_parser_syslog.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,23 @@ def test_parse_with_rfc5424_message_and_without_priority
@parser.instance.patterns['format'])
end

def test_parse_with_rfc5424_empty_message_and_without_priority
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'rfc5424',
)
text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - -'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "-", record["pid"]
assert_equal "-", record["msgid"]
assert_equal "-", record["extradata"]
assert_nil record["message"]
end
assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424,
@parser.instance.patterns['format'])
end

def test_parse_with_rfc5424_message_without_time_format
@parser.configure(
'message_format' => 'rfc5424',
Expand All @@ -133,6 +150,21 @@ def test_parse_with_rfc5424_message_without_time_format
end
end

def test_parse_with_rfc5424_message_with_priority_and_pid
@parser.configure(
'message_format' => 'rfc5424',
'with_priority' => true,
)
text = '<28>1 2018-09-26T15:54:26.620412+09:00 machine minissdpd 1298 - - peer 192.168.0.5:50123 is not from a LAN'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2018-09-26T15:54:26.620412+0900", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "1298", record["pid"]
assert_equal "-", record["msgid"]
assert_equal "-", record["extradata"]
assert_equal " peer 192.168.0.5:50123 is not from a LAN", record["message"]
end
end

def test_parse_with_rfc5424_structured_message
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
Expand All @@ -142,13 +174,64 @@ def test_parse_with_rfc5424_structured_message
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "11111", record["pid"]
assert_equal "11111", record["pid"]
assert_equal "ID24224", record["msgid"]
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end

def test_parse_with_rfc5424_multiple_structured_message
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'rfc5424',
'with_priority' => true,
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"][exampleSDID@20224 class="high"] Hi, from Fluentd!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "11111", record["pid"]
assert_equal "ID24224", record["msgid"]
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"][exampleSDID@20224 class=\"high\"]",
record["extradata"]
assert_equal "Hi, from Fluentd!", record["message"]
end
end

def test_parse_with_rfc5424_message_includes_right_bracket
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'rfc5424',
'with_priority' => true,
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd]!'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "11111", record["pid"]
assert_equal "ID24224", record["msgid"]
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
record["extradata"]
assert_equal "Hi, from Fluentd]!", record["message"]
end
end

def test_parse_with_rfc5424_empty_message
@parser.configure(
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
'message_format' => 'rfc5424',
'with_priority' => true,
)
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"]'
@parser.instance.parse(text) do |time, record|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
assert_equal "11111", record["pid"]
assert_equal "ID24224", record["msgid"]
assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
record["extradata"]
assert_nil record["message"]
end
end
end

class TestAutoRegexp < self
Expand Down

0 comments on commit f4a6f09

Please sign in to comment.