Initial commit

Working plugin. Thanks to AbitMoreDepth for the ruby script (logstash-plugins/logstash-input-syslog#15 (comment))

input-syslog.conf

@@ -0,0 +1,181 @@
+#   Use netcat to send log data
+#   nc <ip> 5514
+
+#   Clear index
+#   curl -XDELETE <ip>:9200/<index>
+
+#   https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.16/administration-guide/8#TOPIC-956415
+#   https://support.oneidentity.com/kb/264126/syslog-message-formats
+#   https://github.com/logstash-plugins/logstash-patterns-core/blob/master/patterns/linux-syslog
+
+input {
+    tcp {
+        port => 5514
+        type => "syslog"
+    }
+    udp {
+        port => 5514
+        type => "syslog"
+    }
+}
+
+filter {
+    if [type] == "syslog" {
+
+        ##
+        # Rename message field to something else to bypass default patterns that
+        # also utilize this field
+        ##
+
+        mutate {
+            rename => {
+                "message" => "full_message"
+            }
+        }
+
+        ##
+        # First grok and check syslog type
+        ##
+
+        # Check if we can match rfc5424
+        if "grokmatch" not in [tags] {
+            grok {
+                match => {
+                    "full_message" => "%{SYSLOG5424LINE}"
+                }
+                add_tag => ["grokmatch", "rfc5424"]
+                add_field => { "syslog_version" => "rfc5424" }
+            }
+        }
+
+        # Try to match rfc3164
+        if "grokmatch" not in [tags] {
+            grok {
+                match => {
+                    "full_message" => "%{SYSLOGLINE}"
+                }
+                add_tag => ["grokmatch", "rfc3164"]
+                remove_tag => [ "_grokparsefailure" ]
+                add_field => { "syslog_version" => "rfc3164" }
+            }
+        }
+
+        # If at the end we don't have a match we tag all the input
+        # so that we can capture it for later analysis
+        if "grokmatch" not in [tags] {
+            mutate {
+                remove_tag => [ "_grokparsefailure" ]
+                add_tag => ["grokmatch", "syslog-mismatch"]
+            }
+        }
+
+        ##
+        # Handle rfc3164
+        #
+        #  <133>Feb 25 14:09:07 webserver syslogd: restarted httpd daemon
+        ##
+
+        if "rfc3164" in [tags] {
+            mutate {
+                rename => {
+                    "logsource" => "host"
+                    "program"   => "process"
+                    # message is already provided by the grok match
+                }
+            }
+
+            date {
+                match => [ "timestamp", "MMM dd HH:mm:ss", "MMM  d HH:mm:ss", "ISO8601" ]
+                remove_field => [ "timestamp" ]
+            }
+
+        }
+
+        ##
+        # Handle rfc5424
+        #
+        # <34>1 2003-10-11T22:14:15.003Z mymachine myapplication 1234 ID47 [example@0 class="high"] BOMmyapplication is started
+        # <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su apache ID47 [exampleSDID@0 iut="3" eventSource="Application" eventID="1011"][examplePriority@0 class="high"] BOM'su root' failed for lonvick on /dev/pts/8
+        ##
+
+        if "rfc5424" in [tags] {
+            mutate {
+                rename => {
+                    "syslog5424_host" => "host"
+                    "syslog5424_app"  => "process"
+                    "syslog5424_msg"  => "message"
+                    "syslog5424_proc" => "process-id"
+                }
+                remove_field => [ "syslog5424_pri", "syslog5424_msgid", "syslog5424_ver" ]
+            }
+
+            date {
+                match => [ "syslog5424_ts", "MMM dd yyyy HH:mm:ss", "MMM  d yyyy HH:mm:ss", "ISO8601" ]
+                remove_field => [ "syslog5424_ts" ]
+            }
+
+            # Pull structured data into params
+
+            if [syslog5424_sd] {
+                ruby {
+                    code => '
+                        # https://github.com/logstash-plugins/logstash-input-syslog/issues/15#issuecomment-270367033
+                        def extract_syslog5424_sd(syslog5424_sd)
+                            sd = {}
+                            syslog5424_sd.scan(/\[(?<element>.*?[^\\])\]/) do |element|
+                                data = element[0].match(/(?<sd_id>[^\ ]+)(?<sd_params> .*)?/)
+                                sd_id = data[:sd_id].split("@", 2)[0]
+                                sd[sd_id] = {}
+                                next if data.nil? || data[:sd_params].nil?
+                                data[:sd_params].scan(/ (.*?[=](?:""|".*?[^\\]"))/) do |set|
+                                    set = set[0].match(/(?<param_name>.*?)[=]\"(?<param_value>.*)\"/)
+                                    sd[sd_id][set[:param_name]] = set[:param_value]
+                                end
+                            end
+                            sd
+                        end
+                        event.set("[structured_data]", extract_syslog5424_sd(event.get("[syslog5424_sd]")))
+                    '
+                    remove_field => "syslog5424_sd"
+                }
+            }
+
+        }
+
+        ##
+        # Cleanup
+        ##
+
+        mutate {
+            remove_field => [ "port" ]
+        }
+
+    }
+}
+
+output {
+    # Split all types of syslog to a seperate index
+    if [type] == "syslog" and "rfc5424" in [tags] {
+        elasticsearch {
+            hosts => ["localhost:9200"]
+            index => "syslog-rfc5424-%{+YYYY.MM.dd}"
+        }
+    }
+
+    if [type] == "syslog" and "rfc3164" in [tags] {
+        elasticsearch {
+            hosts => ["localhost:9200"]
+            index => "syslog-rfc3164-%{+YYYY.MM.dd}"
+        }
+    }
+
+    # Mismatched capture
+    if [type] == "syslog" and "syslog-mismatch" in [tags] {
+        file {
+            path => "/var/log/syslog-mismatch-%{+YYYY-MM-dd}"
+        }
+    }
+
+    # Standard output to console for debugging
+    # stdout { codec => rubydebug }
+}