From 50c83844a76006f6346f40d0885e7bf55914e8b9 Mon Sep 17 00:00:00 2001 From: ruflin Date: Fri, 26 Aug 2016 17:59:13 +0200 Subject: [PATCH] Move ignore_outgoing to packetbeat * Update documentation * Update CHANGELOG * Remove option from test config files as not needed * Add option to migration script --- CHANGELOG.asciidoc | 1 + filebeat/filebeat.full.yml | 5 --- filebeat/tests/system/config/filebeat.yml.j2 | 5 --- .../config/{{cookiecutter.beat}}.yml.j2 | 29 ++++++--------- libbeat/_meta/config.full.yml | 5 --- libbeat/docs/generalconfig.asciidoc | 37 ------------------- libbeat/publisher/publish.go | 7 ---- .../scripts/migrate_beat_config_1_x_to_5_0.py | 9 +++-- libbeat/tests/system/config/mockbeat.yml.j2 | 6 --- metricbeat/metricbeat.full.yml | 5 --- .../tests/system/config/metricbeat.yml.j2 | 5 --- packetbeat/beater/packetbeat.go | 3 +- packetbeat/config/config.go | 11 +++--- .../configuration/packetbeat-options.asciidoc | 32 ++++++++++++++++ packetbeat/etc/beat.full.yml | 4 ++ packetbeat/packetbeat.full.yml | 9 ++--- packetbeat/publish/publish.go | 4 +- packetbeat/publish/publish_test.go | 6 +-- .../tests/system/config/packetbeat.yml.j2 | 5 --- winlogbeat/config/config.go | 2 +- winlogbeat/config/config_test.go | 2 +- winlogbeat/winlogbeat.full.yml | 5 --- 22 files changed, 74 insertions(+), 123 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index bc13a2622629..e1e7c86cfa00 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -24,6 +24,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha5...master[Check the HEAD d *Packetbeat* - Group HTTP fields under `http.request` and `http.response` {pull}2167[2167] - Export `http.request.body` and `http.response.body` when configured under `include_body_for` {pull}2167[2167] +- Move `ignore_outgoing` config to `packetbeat.ignore_outgoing` {pull}2393[2393] *Topbeat* diff --git a/filebeat/filebeat.full.yml b/filebeat/filebeat.full.yml index 4e1bf7fa4443..262c6ab3e0ce 100644 --- a/filebeat/filebeat.full.yml +++ b/filebeat/filebeat.full.yml @@ -242,11 +242,6 @@ filebeat.prospectors: # sub-dictionary. Default is false. #fields_under_root: false -# Uncomment the following if you want to ignore transactions created -# by the server on which the shipper is installed. This option is useful -# to remove duplicates if shippers are installed on multiple servers. -#ignore_outgoing: true - # How often (in seconds) shippers are publishing their IPs to the topology map. # The default is 10 seconds. #refresh_topology_freq: 10 diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index cafcc58c5523..483803c9fe6b 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -93,11 +93,6 @@ tags: [ {%- endif -%} ] -# Uncomment the following if you want to ignore transactions created -# by the server on which the shipper is installed. This option is useful -# to remove duplicates if shippers are installed on multiple servers. -# ignore_outgoing: true - {% if geoip_paths is not none %} geoip: paths: [ diff --git a/generate/beat/{{cookiecutter.beat}}/tests/system/config/{{cookiecutter.beat}}.yml.j2 b/generate/beat/{{cookiecutter.beat}}/tests/system/config/{{cookiecutter.beat}}.yml.j2 index e834a5b5de9e..e95072740678 100644 --- a/generate/beat/{{cookiecutter.beat}}/tests/system/config/{{cookiecutter.beat}}.yml.j2 +++ b/generate/beat/{{cookiecutter.beat}}/tests/system/config/{{cookiecutter.beat}}.yml.j2 @@ -32,23 +32,18 @@ output: -############################# Shipper ######################################### - -shipper: - # The name of the shipper that publishes the network data. It can be used to group - # all the transactions sent by a single shipper in the web interface. - # If this options is not defined, the hostname is used. - #name: - - # The tags of the shipper are included in their own field with each - # transaction published. Tags make it easy to group servers by different - # logical properties. - #tags: ["service-X", "web-tier"] - - # Uncomment the following if you want to ignore transactions created - # by the server on which the shipper is installed. This option is useful - # to remove duplicates if shippers are installed on multiple servers. - #ignore_outgoing: true +############################# Beat ######################################### + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + ############################# Logging ######################################### diff --git a/libbeat/_meta/config.full.yml b/libbeat/_meta/config.full.yml index 448cda05e718..9f2dd3fb245f 100644 --- a/libbeat/_meta/config.full.yml +++ b/libbeat/_meta/config.full.yml @@ -22,11 +22,6 @@ # sub-dictionary. Default is false. #fields_under_root: false -# Uncomment the following if you want to ignore transactions created -# by the server on which the shipper is installed. This option is useful -# to remove duplicates if shippers are installed on multiple servers. -#ignore_outgoing: true - # How often (in seconds) shippers are publishing their IPs to the topology map. # The default is 10 seconds. #refresh_topology_freq: 10 diff --git a/libbeat/docs/generalconfig.asciidoc b/libbeat/docs/generalconfig.asciidoc index 2a4baecc1599..422eafe2beba 100644 --- a/libbeat/docs/generalconfig.asciidoc +++ b/libbeat/docs/generalconfig.asciidoc @@ -30,11 +30,6 @@ Here is an example configuration: # logical properties. tags: ["service-X", "web-tier"] -# Uncomment the following if you want to ignore transactions created -# by the server on which the shipper is installed. This option is useful -# to remove duplicates if shippers are installed on multiple servers. -ignore_outgoing: true - # How often (in seconds) shippers are publishing their IPs to the topology map. # The default is 10 seconds. refresh_topology_freq: 10 @@ -124,38 +119,6 @@ fields: region: us-east-1 ------------------------------------------------------------------------------ -===== ignore_outgoing - -If the `ignore_outgoing` option is enabled, the Beat ignores all the -transactions initiated from the server running the Beat. - -This is useful when two Beats publish the same transactions. Because one Beat -sees the transaction in its outgoing queue and the other sees it in its incoming -queue, you can end up with duplicate transactions. To remove the duplicates, you -can enable the `ignore_outgoing` option on one of the servers. - -For example, in the following scenario, you see a 3-server architecture -where a Beat is installed on each server. t1 is the transaction exchanged between -Server1 and Server2, and t2 is the transaction between Server2 and Server3. - -image:./images/option_ignore_outgoing.png[Beats Architecture] - -By default, each transaction is indexed twice because Beat2 -sees both transactions. So you would see the following published transactions -(when `ignore_outgoing` is false): - - - Beat1: t1 - - Beat2: t1 and t2 - - Beat3: t2 - -To avoid duplicates, you can force your Beats to send only the incoming -transactions and ignore the transactions created by the local server. So you would -see the following published transactions (when `ignore_outgoing` is true): - - - Beat1: none - - Beat2: t1 - - Beat3: t2 - ===== refresh_topology_freq The refresh interval of the topology map in diff --git a/libbeat/publisher/publish.go b/libbeat/publisher/publish.go index 9a3da46bb35a..00790b874e84 100644 --- a/libbeat/publisher/publish.go +++ b/libbeat/publisher/publish.go @@ -59,7 +59,6 @@ type BeatPublisher struct { Index string Output []*outputWorker TopologyOutput outputs.TopologyOutputer - ignoreOutgoing bool geoLite *libgeo.GeoIP Processors *processors.Processors @@ -87,7 +86,6 @@ type ShipperConfig struct { common.EventMetadata `config:",inline"` // Fields and tags to add to each event. Name string `config:"name"` RefreshTopologyFreq time.Duration `config:"refresh_topology_freq"` - Ignore_outgoing bool `config:"ignore_outgoing"` Topology_expire int `config:"topology_expire"` Geoip common.Geoip `config:"geoip"` @@ -145,10 +143,6 @@ func (publisher *BeatPublisher) GeoLite() *libgeo.GeoIP { return publisher.geoLite } -func (publisher *BeatPublisher) IgnoreOutgoing() bool { - return publisher.ignoreOutgoing -} - func (publisher *BeatPublisher) Connect() Client { atomic.AddUint32(&publisher.numClients, 1) return newClient(publisher) @@ -207,7 +201,6 @@ func (publisher *BeatPublisher) init( processors *processors.Processors, ) error { var err error - publisher.ignoreOutgoing = shipper.Ignore_outgoing publisher.Processors = processors publisher.disabled = *publishDisabled diff --git a/libbeat/scripts/migrate_beat_config_1_x_to_5_0.py b/libbeat/scripts/migrate_beat_config_1_x_to_5_0.py index 008f3c5b72c3..f65c7219281f 100644 --- a/libbeat/scripts/migrate_beat_config_1_x_to_5_0.py +++ b/libbeat/scripts/migrate_beat_config_1_x_to_5_0.py @@ -7,7 +7,7 @@ def migrate_packetbeat(content): Changes things like `interfaces:` to `packetbeat.interfaces:` at the top level. """ - sections = ["interfaces", "protocols", "procs", "runoptions"] + sections = ["interfaces", "protocols", "procs", "runoptions", "ignore_outgoing"] lines = content.splitlines() outlines = [] for line in lines: @@ -61,8 +61,9 @@ def main(): with open(args.file, "r") as f: content = f.read() - out = migrate_packetbeat(content) - out = migrate_shipper(out) + # Shipper must be migrated first for ignore_outgoing to be applied properly + out = migrate_shipper(content) + out = migrate_packetbeat(out) if args.dry: print(out) @@ -92,6 +93,7 @@ def test_migrate_packetbeat(): ports: [53] runoptions: procs: +ignore_outgoing: true """ output = migrate_packetbeat(test) @@ -109,6 +111,7 @@ def test_migrate_packetbeat(): ports: [53] packetbeat.runoptions: packetbeat.procs: +packetbeat.ignore_outgoing: true """ diff --git a/libbeat/tests/system/config/mockbeat.yml.j2 b/libbeat/tests/system/config/mockbeat.yml.j2 index 3048818e35ff..ce4d5bfc881e 100644 --- a/libbeat/tests/system/config/mockbeat.yml.j2 +++ b/libbeat/tests/system/config/mockbeat.yml.j2 @@ -23,12 +23,6 @@ tags: [ {%- endif -%}] -# Uncomment the following if you want to ignore transactions created -# by the server on which the shipper is installed. This option is useful -# to remove duplicates if shippers are installed on multiple servers. -# ignore_outgoing: true - - ############################# Output ############################################ diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 5f72dc5b512f..4806f76675f3 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -206,11 +206,6 @@ metricbeat.modules: # sub-dictionary. Default is false. #fields_under_root: false -# Uncomment the following if you want to ignore transactions created -# by the server on which the shipper is installed. This option is useful -# to remove duplicates if shippers are installed on multiple servers. -#ignore_outgoing: true - # How often (in seconds) shippers are publishing their IPs to the topology map. # The default is 10 seconds. #refresh_topology_freq: 10 diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index 264f3310f6b8..bad37431ef72 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -87,11 +87,6 @@ tags: [ {%- endif -%} ] -# Uncomment the following if you want to ignore transactions created -# by the server on which the shipper is installed. This option is useful -# to remove duplicates if shippers are installed on multiple servers. -# ignore_outgoing: true - {% if geoip_paths is not none %} geoip: paths: [ diff --git a/packetbeat/beater/packetbeat.go b/packetbeat/beater/packetbeat.go index 7b5c859103be..7a01a2be474f 100644 --- a/packetbeat/beater/packetbeat.go +++ b/packetbeat/beater/packetbeat.go @@ -111,7 +111,8 @@ func (pb *Packetbeat) init(b *beat.Beat) error { if b.Config.Shipper.BulkQueueSize != nil { bulkQueueSize = *b.Config.Shipper.BulkQueueSize } - pb.Pub, err = publish.NewPublisher(b.Publisher, queueSize, bulkQueueSize) + + pb.Pub, err = publish.NewPublisher(b.Publisher, queueSize, bulkQueueSize, pb.Config.IgnoreOutgoing) if err != nil { return fmt.Errorf("Initializing publisher failed: %v", err) } diff --git a/packetbeat/config/config.go b/packetbeat/config/config.go index f02ffc5087a3..12c7f120d394 100644 --- a/packetbeat/config/config.go +++ b/packetbeat/config/config.go @@ -9,11 +9,12 @@ import ( ) type Config struct { - Interfaces InterfacesConfig `config:"interfaces"` - Flows *Flows `config:"flows"` - Protocols map[string]*common.Config `config:"protocols"` - Procs procs.ProcsConfig `config:"procs"` - RunOptions droppriv.RunOptions + Interfaces InterfacesConfig `config:"interfaces"` + Flows *Flows `config:"flows"` + Protocols map[string]*common.Config `config:"protocols"` + Procs procs.ProcsConfig `config:"procs"` + IgnoreOutgoing bool `config:"ignore_outgoing"` + RunOptions droppriv.RunOptions } type InterfacesConfig struct { diff --git a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc index ab900c9c16f6..cef30b2e73bb 100644 --- a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc +++ b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc @@ -169,6 +169,38 @@ NOTE: This setting disables automatic generation of the BPF filter. If you use this setting, it's your responsibility to keep the BPF filters in sync with the ports defined in the `protocols` section. +===== ignore_outgoing + +If the `ignore_outgoing` option is enabled, Packetbeat ignores all the +transactions initiated from the server running Packetbeat. + +This is useful when two Packetbeat instances publish the same transactions. Because one Packetbeat +sees the transaction in its outgoing queue and the other sees it in its incoming +queue, you can end up with duplicate transactions. To remove the duplicates, you +can enable the `packetbeat.ignore_outgoing` option on one of the servers. + +For example, in the following scenario, you see a 3-server architecture +where a Beat is installed on each server. t1 is the transaction exchanged between +Server1 and Server2, and t2 is the transaction between Server2 and Server3. + +image:./images/option_ignore_outgoing.png[Beats Architecture] + +By default, each transaction is indexed twice because Beat2 +sees both transactions. So you would see the following published transactions +(when `ignore_outgoing` is false): + + - Beat1: t1 + - Beat2: t1 and t2 + - Beat3: t2 + +To avoid duplicates, you can force your Beats to send only the incoming +transactions and ignore the transactions created by the local server. So you would +see the following published transactions (when `ignore_outgoing` is true): + + - Beat1: none + - Beat2: t1 + - Beat3: t2 + [[configuration-flows]] === Flows Configuration diff --git a/packetbeat/etc/beat.full.yml b/packetbeat/etc/beat.full.yml index 4c4f334cf9cf..079e4414bdce 100644 --- a/packetbeat/etc/beat.full.yml +++ b/packetbeat/etc/beat.full.yml @@ -444,3 +444,7 @@ packetbeat.protocols.nfs: # - process: app # cmdline_grep: gunicorn +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +#packetbeat.ignore_outgoing: true diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index 07d59eba001e..fbf39e9a21ba 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -444,6 +444,10 @@ packetbeat.protocols.nfs: # - process: app # cmdline_grep: gunicorn +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +#packetbeat.ignore_outgoing: true #================================ General ===================================== @@ -468,11 +472,6 @@ packetbeat.protocols.nfs: # sub-dictionary. Default is false. #fields_under_root: false -# Uncomment the following if you want to ignore transactions created -# by the server on which the shipper is installed. This option is useful -# to remove duplicates if shippers are installed on multiple servers. -#ignore_outgoing: true - # How often (in seconds) shippers are publishing their IPs to the topology map. # The default is 10 seconds. #refresh_topology_freq: 10 diff --git a/packetbeat/publish/publish.go b/packetbeat/publish/publish.go index 151b8f3613f4..b2775c078265 100644 --- a/packetbeat/publish/publish.go +++ b/packetbeat/publish/publish.go @@ -45,7 +45,6 @@ type TopologyProvider interface { IsPublisherIP(ip string) bool GetServerName(ip string) string GeoLite() *libgeo.GeoIP - IgnoreOutgoing() bool } func (t *ChanTransactions) PublishTransaction(event common.MapStr) bool { @@ -58,6 +57,7 @@ var debugf = logp.MakeDebug("publish") func NewPublisher( pub publisher.Publisher, hwm, bulkHWM int, + ignoreOutgoing bool, ) (*PacketbeatPublisher, error) { topo, ok := pub.(TopologyProvider) if !ok { @@ -68,7 +68,7 @@ func NewPublisher( pub: pub, topo: topo, geoLite: topo.GeoLite(), - ignoreOutgoing: topo.IgnoreOutgoing(), + ignoreOutgoing: ignoreOutgoing, client: pub.Connect(), done: make(chan struct{}), trans: make(chan common.MapStr, hwm), diff --git a/packetbeat/publish/publish_test.go b/packetbeat/publish/publish_test.go index cd2d2c2ebae9..a4540ef5ac34 100644 --- a/packetbeat/publish/publish_test.go +++ b/packetbeat/publish/publish_test.go @@ -63,7 +63,7 @@ func TestFilterEvent(t *testing.T) { func TestDirectionOut(t *testing.T) { publisher := newTestPublisher([]string{"192.145.2.4"}) - ppub, _ := NewPublisher(publisher, 1000, 1) + ppub, _ := NewPublisher(publisher, 1000, 1, false) event := common.MapStr{ "src": &common.Endpoint{ @@ -89,7 +89,7 @@ func TestDirectionOut(t *testing.T) { func TestDirectionIn(t *testing.T) { publisher := newTestPublisher([]string{"192.145.2.5"}) - ppub, _ := NewPublisher(publisher, 1000, 1) + ppub, _ := NewPublisher(publisher, 1000, 1, false) event := common.MapStr{ "src": &common.Endpoint{ @@ -121,7 +121,7 @@ func newTestPublisher(ips []string) *publisher.BeatPublisher { func TestNoDirection(t *testing.T) { publisher := newTestPublisher([]string{"192.145.2.6"}) - ppub, _ := NewPublisher(publisher, 1000, 1) + ppub, _ := NewPublisher(publisher, 1000, 1, false) event := common.MapStr{ "src": &common.Endpoint{ diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index 3a8ec4959dba..79db5b9bf83d 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -168,11 +168,6 @@ tags: [ {%- endif -%} ] -# Uncomment the following if you want to ignore transactions created -# by the server on which the shipper is installed. This option is useful -# to remove duplicates if shippers are installed on multiple servers. -# ignore_outgoing: true - {% if geoip_paths is not none %} geoip: paths: [ diff --git a/winlogbeat/config/config.go b/winlogbeat/config/config.go index 4681d4e17bfd..c73cf52d8695 100644 --- a/winlogbeat/config/config.go +++ b/winlogbeat/config/config.go @@ -46,7 +46,7 @@ func (s Settings) Validate() error { validKeys := []string{ "fields", "fields_under_root", "tags", - "name", "refresh_topology_freq", "ignore_outgoing", "topology_expire", "geoip", + "name", "refresh_topology_freq", "topology_expire", "geoip", "queue_size", "bulk_queue_size", "max_procs", "processors", "logging", "output", "path", "winlogbeat", } diff --git a/winlogbeat/config/config_test.go b/winlogbeat/config/config_test.go index 2bb9e470bc83..0c39a2610387 100644 --- a/winlogbeat/config/config_test.go +++ b/winlogbeat/config/config_test.go @@ -45,7 +45,7 @@ func TestConfigValidate(t *testing.T) { map[string]interface{}{"other": "value"}, }, "1 error: Invalid top-level key 'other' found. Valid keys are bulk_queue_size, " + - "fields, fields_under_root, geoip, ignore_outgoing, logging, max_procs, " + + "fields, fields_under_root, geoip, logging, max_procs, " + "name, output, path, processors, queue_size, refresh_topology_freq, tags, topology_expire, winlogbeat", }, { diff --git a/winlogbeat/winlogbeat.full.yml b/winlogbeat/winlogbeat.full.yml index 2211f9e4d801..6f8ee00845a2 100644 --- a/winlogbeat/winlogbeat.full.yml +++ b/winlogbeat/winlogbeat.full.yml @@ -57,11 +57,6 @@ winlogbeat.event_logs: # sub-dictionary. Default is false. #fields_under_root: false -# Uncomment the following if you want to ignore transactions created -# by the server on which the shipper is installed. This option is useful -# to remove duplicates if shippers are installed on multiple servers. -#ignore_outgoing: true - # How often (in seconds) shippers are publishing their IPs to the topology map. # The default is 10 seconds. #refresh_topology_freq: 10