diff --git a/filebeat/input/syslog/config.go b/filebeat/input/syslog/config.go index 76c4480c12c4..ff2b68dea30c 100644 --- a/filebeat/input/syslog/config.go +++ b/filebeat/input/syslog/config.go @@ -21,8 +21,6 @@ import ( "fmt" "time" - "github.com/dustin/go-humanize" - "github.com/elastic/beats/filebeat/harvester" "github.com/elastic/beats/filebeat/inputsource" "github.com/elastic/beats/filebeat/inputsource/tcp" @@ -30,9 +28,26 @@ import ( "github.com/elastic/beats/libbeat/common" ) +// ConditionConfig : 用于条件表达式，目前支持=、!=、eq、neq、include、exclude、regex、nregex +type ConditionConfig struct { + Key string `config:"key"` + Op string `config:"op"` + Value string `config:"value"` + matcher MatchFunc +} + +func (c *ConditionConfig) GetMatcher() MatchFunc { + return c.matcher +} + +type FilterConfig struct { + Conditions []ConditionConfig `config:"conditions"` +} + type config struct { harvester.ForwarderConfig `config:",inline"` Protocol common.ConfigNamespace `config:"protocol"` + SyslogFilters []FilterConfig `config:"syslog_filters"` } var defaultConfig = config{ diff --git a/filebeat/input/syslog/input.go b/filebeat/input/syslog/input.go index 1f5e29decc10..e3748be7248b 100644 --- a/filebeat/input/syslog/input.go +++ b/filebeat/input/syslog/input.go @@ -129,6 +129,162 @@ func syslogFormatter(event *util.Data) *util.Data { return event } +type SyslogFields struct { + Data interface{} + Address interface{} + Port interface{} + Facility interface{} + FacilityLabel interface{} + Priority interface{} + Severity interface{} + SeverityLabel interface{} + Program interface{} + Pid interface{} +} + +func (s *SyslogFields) GetSyslogFieldValue(key string) interface{} { + key = strings.ToLower(key) + switch key { + case "data", "": + return s.Data + case "address": + return s.Address + case "port": + return s.Port + case "facility": + return s.Facility + case "facility_label": + return s.FacilityLabel + case "priority": + return s.Priority + case "severity": + return s.Severity + case "severity_label": + return s.SeverityLabel + case "program": + return s.Program + case "pid": + return s.Pid + default: + return "" + } +} + +func flattenMap(original common.MapStr, flatMap common.MapStr) { + for key, value := range original { + fullKey := key + if subMap, ok := value.(common.MapStr); ok { + flattenMap(subMap, flatMap) + } else { + flatMap[fullKey] = value + } + } +} + +func parseSyslogField(data *util.Data) *SyslogFields { + + flatMap := make(common.MapStr) + + flattenMap(data.Event.Fields, flatMap) + + // 解析 syslog 字段值 + syslogFields := &SyslogFields{ + Data: flatMap["data"], + Address: flatMap["address"], + Port: flatMap["port"], + Facility: flatMap["facility"], + FacilityLabel: flatMap["facility_label"], + Priority: flatMap["priority"], + Severity: flatMap["severity"], + SeverityLabel: flatMap["severity_label"], + Program: flatMap["program"], + Pid: flatMap["pid"], + } + return syslogFields +} + +// Filter Syslog filter +func Filter(data *util.Data, config *config) bool { + + var text string + var ok bool + + event := &data.Event + text, ok = event.Fields["data"].(string) + + if !ok { + return false + } + log := logp.NewLogger("syslog") + + syslogFields := parseSyslogField(data) + + for _, filterConfig := range config.SyslogFilters { + access := true + for _, condition := range filterConfig.Conditions { + + matcher := condition.GetMatcher() + if matcher == nil { + access = false + break + } + + // syslog field value match + fieldValue := syslogFields.GetSyslogFieldValue(condition.Key) + switch v := fieldValue.(type) { + case string: + text = v + case int: + text = strconv.Itoa(v) + default: + text = "" + } + + if text == "" { + access = false + break + } + + if !matcher(text) { + access = false + break + } else { + continue + } + } + if access { + return true + } + } + return false +} + +func InitFilterMatcher(config *config) (*config, error) { + if config.SyslogFilters == nil { + return config, nil + } + for _, f := range config.SyslogFilters { + for i, condition := range f.Conditions { + // 去除字符串首尾空白字符 + condition.Key = strings.TrimSpace(condition.Key) + condition.Value = strings.TrimSpace(condition.Value) + + // 初始化条件匹配方法 Matcher + matcher, err := getOperationFunc(condition.Op, condition.Value) + + if err != nil { + return nil, err + } + + condition.matcher = matcher + + // 重新赋值 condition + f.Conditions[i] = condition + } + } + return config, nil +} + // Input define a syslog input type Input struct { sync.Mutex @@ -159,6 +315,12 @@ func NewInput( return nil, err } + conf, err := InitFilterMatcher(&config) + if err != nil { + return nil, err + } + config = *conf + forwarder := harvester.NewForwarder(out) cb := func(data []byte, metadata inputsource.NetworkMetadata) { ev := newEvent() @@ -184,7 +346,13 @@ func NewInput( d = &util.Data{Event: *event} } d = syslogFormatter(d) - forwarder.Send(d) + filterAccess := true + if config.SyslogFilters != nil { + filterAccess = Filter(d, &config) + } + if filterAccess { + forwarder.Send(d) + } } server, err := factory(cb, config.Protocol) diff --git a/filebeat/input/syslog/operator.go b/filebeat/input/syslog/operator.go new file mode 100644 index 000000000000..081ce6ac122d --- /dev/null +++ b/filebeat/input/syslog/operator.go @@ -0,0 +1,111 @@ +// Tencent is pleased to support the open source community by making bkunifylogbeat 蓝鲸日志采集器 available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +// +// bkunifylogbeat 蓝鲸日志采集器 is licensed under the MIT License. +// +// License for bkunifylogbeat 蓝鲸日志采集器: +// -------------------------------------------------------------------- +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +// and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or substantial +// portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +// LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package syslog + +import ( + "fmt" + "regexp" + "strings" +) + +type MatchFunc func(text string) bool + +func equal(value string) (MatchFunc, error) { + matcher := func(text string) bool { + return text == value + } + return matcher, nil +} + +func notEqual(value string) (MatchFunc, error) { + matcher := func(text string) bool { + return text != value + } + return matcher, nil +} + +func include(value string) (MatchFunc, error) { + matcher := func(text string) bool { + return strings.Contains(text, value) + } + return matcher, nil +} + +func exclude(value string) (MatchFunc, error) { + matcher := func(text string) bool { + return !strings.Contains(text, value) + } + return matcher, nil +} + +func regex(value string) (MatchFunc, error) { + pattern, err := regexp.Compile(value) + if err != nil { + return nil, err + } + matcher := func(text string) bool { + return pattern.MatchString(text) + } + return matcher, nil +} + +func nRegex(value string) (MatchFunc, error) { + pattern, err := regexp.Compile(value) + if err != nil { + return nil, err + } + matcher := func(text string) bool { + return !pattern.MatchString(text) + } + return matcher, nil +} + +const ( + opEq = "eq" + opNeq = "neq" + opEqual = "=" + opNotEqual = "!=" + opInclude = "include" + opExclude = "exclude" + opRegex = "regex" + opNregex = "nregex" +) + +func getOperationFunc(op string, value string) (MatchFunc, error) { + switch op { + case opEqual, opEq: + return equal(value) + case opNotEqual, opNeq: + return notEqual(value) + case opInclude: + return include(value) + case opExclude: + return exclude(value) + case opRegex: + return regex(value) + case opNregex: + return nRegex(value) + default: + return nil, fmt.Errorf("op:[%s] is not supported", op) + } +}