From 4e4b3e5af23fecc4f7fc1ed1dd74346d93e9a863 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 26 Sep 2018 10:36:18 +0900
Subject: [PATCH 1/4] Add test cases according to #2068

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 test/plugin/test_parser_syslog.rb | 34 +++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb
index 7373000138..f5d0b7736e 100644
--- a/test/plugin/test_parser_syslog.rb
+++ b/test/plugin/test_parser_syslog.rb
@@ -149,6 +149,40 @@ def test_parse_with_rfc5424_structured_message
         assert_equal "Hi, from Fluentd!", record["message"]
       end
     end
+
+    def test_parse_with_rfc5424_multiple_structured_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"][exampleSDID@20224 class="high"] Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"][exampleSDID@20224 class=\"high\"]",
+                     record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+
+    def test_parse_with_rfc5424_message_includes_right_bracket
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd]!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                     record["extradata"]
+        assert_equal "Hi, from Fluentd]!", record["message"]
+      end
+    end
   end
 
   class TestAutoRegexp < self

From abcb73b8c004232bb8a2a6544a349bc75c471252 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 26 Sep 2018 10:45:24 +0900
Subject: [PATCH 2/4] Support empty message

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 lib/fluent/plugin/parser_syslog.rb |  4 ++--
 test/plugin/test_parser_syslog.rb  | 34 ++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb
index 954df0fa2f..998828430c 100644
--- a/lib/fluent/plugin/parser_syslog.rb
+++ b/lib/fluent/plugin/parser_syslog.rb
@@ -27,8 +27,8 @@ class SyslogParser < Parser
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
-      REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>.{1,128}) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
-      REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>.{1,128}) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
+      REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>.{1,128}) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ]))(?: (?<message>.+))?$\z/
+      REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>.{1,128}) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ]))(?: (?<message>.+))?$\z/
       REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb
index f5d0b7736e..fa2126c8ce 100644
--- a/test/plugin/test_parser_syslog.rb
+++ b/test/plugin/test_parser_syslog.rb
@@ -118,6 +118,23 @@ def test_parse_with_rfc5424_message_and_without_priority
                    @parser.instance.patterns['format'])
     end
 
+    def test_parse_with_rfc5424_empty_message_and_without_priority
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        )
+      text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - -'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_nil record["message"]
+      end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424,
+                   @parser.instance.patterns['format'])
+    end
+
     def test_parse_with_rfc5424_message_without_time_format
       @parser.configure(
                         'message_format' => 'rfc5424',
@@ -183,6 +200,23 @@ def test_parse_with_rfc5424_message_includes_right_bracket
         assert_equal "Hi, from Fluentd]!", record["message"]
       end
     end
+
+    def test_parse_with_rfc5424_empty_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"]'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                     record["extradata"]
+        assert_nil record["message"]
+      end
+    end
   end
 
   class TestAutoRegexp < self

From 0b5a3b02323aae1e1ea67210fa8d98808faf3d9e Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 26 Sep 2018 16:21:50 +0900
Subject: [PATCH 3/4] Fix regexp for RFC5424 to parse message includes PID
 properly

In previous version, pid field is too greedy.

See also https://tools.ietf.org/html/rfc5424#section-6 and #2068

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 lib/fluent/plugin/parser_syslog.rb |  4 ++--
 test/plugin/test_parser_syslog.rb  | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb
index 998828430c..778818a145 100644
--- a/lib/fluent/plugin/parser_syslog.rb
+++ b/lib/fluent/plugin/parser_syslog.rb
@@ -27,8 +27,8 @@ class SyslogParser < Parser
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
-      REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>.{1,128}) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ]))(?: (?<message>.+))?$\z/
-      REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>.{1,128}) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ]))(?: (?<message>.+))?$\z/
+      REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
+      REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
       REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb
index fa2126c8ce..722ffa40af 100644
--- a/test/plugin/test_parser_syslog.rb
+++ b/test/plugin/test_parser_syslog.rb
@@ -150,6 +150,21 @@ def test_parse_with_rfc5424_message_without_time_format
       end
     end
 
+    def test_parse_with_rfc5424_message_with_priority_and_pid
+      @parser.configure(
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        )
+      text = '<28>1 2018-09-26T15:54:26.620412+09:00 machine minissdpd 1298 - -  peer 192.168.0.5:50123 is not from a LAN'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2018-09-26T15:54:26.620412+0900", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "1298", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal " peer 192.168.0.5:50123 is not from a LAN", record["message"]
+      end
+    end
+
     def test_parse_with_rfc5424_structured_message
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',

From d9810b4fc9e11b770379a1398b88b7c867ac6a59 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 26 Sep 2018 16:51:43 +0900
Subject: [PATCH 4/4] Fix indent

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 test/plugin/test_parser_syslog.rb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb
index 722ffa40af..1d53df5397 100644
--- a/test/plugin/test_parser_syslog.rb
+++ b/test/plugin/test_parser_syslog.rb
@@ -174,7 +174,7 @@ def test_parse_with_rfc5424_structured_message
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-assert_equal "11111", record["pid"]
+        assert_equal "11111", record["pid"]
         assert_equal "ID24224", record["msgid"]
         assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
                      record["extradata"]
@@ -191,7 +191,7 @@ def test_parse_with_rfc5424_multiple_structured_message
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"][exampleSDID@20224 class="high"] Hi, from Fluentd!'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-assert_equal "11111", record["pid"]
+        assert_equal "11111", record["pid"]
         assert_equal "ID24224", record["msgid"]
         assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"][exampleSDID@20224 class=\"high\"]",
                      record["extradata"]
@@ -208,7 +208,7 @@ def test_parse_with_rfc5424_message_includes_right_bracket
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd]!'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-assert_equal "11111", record["pid"]
+        assert_equal "11111", record["pid"]
         assert_equal "ID24224", record["msgid"]
         assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
                      record["extradata"]
@@ -225,7 +225,7 @@ def test_parse_with_rfc5424_empty_message
       text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"]'
       @parser.instance.parse(text) do |time, record|
         assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-assert_equal "11111", record["pid"]
+        assert_equal "11111", record["pid"]
         assert_equal "ID24224", record["msgid"]
         assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
                      record["extradata"]