From 4aca334c9b07e3ac7bf6427a09b6bd6638f525fe Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 23 Nov 2020 15:14:24 -0700 Subject: [PATCH 1/5] drop aws.vpcflow.pkt_dstaddr and aws.vpcflow.pkt_srcaddr when equals to "-" --- .../module/aws/vpcflow/config/input.yml | 8 +++ .../test/tcp-flag-sequence-skip-data.log | 5 ++ ...-flag-sequence-skip-data.log-expected.json | 64 +++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log create mode 100644 x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml index de4affbd694c..43ec062acfe2 100644 --- a/x-pack/filebeat/module/aws/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/aws/vpcflow/config/input.yml @@ -129,6 +129,14 @@ processors: - drop_fields: fields: ["aws.vpcflow.srcaddr", "aws.vpcflow.srcport", "aws.vpcflow.dstaddr", "aws.vpcflow.dstport", "aws.vpcflow.bytes", "aws.vpcflow.packets", "aws.vpcflow.protocol"] + - drop_fields: + when.equals.aws.vpcflow.pkt_srcaddr: "-" + fields: ["aws.vpcflow.pkt_srcaddr"] + + - drop_fields: + when.equals.aws.vpcflow.pkt_dstaddr: "-" + fields: [ "aws.vpcflow.pkt_dstaddr" ] + - community_id: ~ # Use the aws.vpcflow.action value to set the event.outcome value to either "allow" or "deny". diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log new file mode 100644 index 000000000000..2ce24460ff9f --- /dev/null +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log @@ -0,0 +1,5 @@ +version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status +3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA + +version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status +3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json new file mode 100644 index 000000000000..aa9cc75cfe73 --- /dev/null +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json @@ -0,0 +1,64 @@ +[ + { + "@timestamp": "2019-08-26T19:48:53.000Z", + "aws.vpcflow.account_id": "123456789010", + "aws.vpcflow.action": "-", + "aws.vpcflow.instance_id": "i-01234567890123456", + "aws.vpcflow.interface_id": "eni-1235b8ca123456789", + "aws.vpcflow.log_status": "SKIPDATA", + "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", + "aws.vpcflow.tcp_flags": "-", + "aws.vpcflow.type": "-", + "aws.vpcflow.version": "3", + "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", + "cloud.account.id": "123456789010", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", + "event.category": "network_traffic", + "event.dataset": "aws.vpcflow", + "event.end": "2019-08-26T19:48:53.000Z", + "event.kind": "event", + "event.module": "aws", + "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA", + "event.start": "2019-08-26T19:47:55.000Z", + "event.type": "flow", + "fileset.name": "vpcflow", + "input.type": "log", + "log.offset": 183, + "service.type": "aws", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2019-08-26T19:48:53.000Z", + "aws.vpcflow.account_id": "123456789010", + "aws.vpcflow.action": "-", + "aws.vpcflow.instance_id": "i-01234567890123456", + "aws.vpcflow.interface_id": "eni-1235b8ca123456789", + "aws.vpcflow.log_status": "NODATA", + "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", + "aws.vpcflow.tcp_flags": "-", + "aws.vpcflow.type": "-", + "aws.vpcflow.version": "3", + "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", + "cloud.account.id": "123456789010", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", + "event.category": "network_traffic", + "event.dataset": "aws.vpcflow", + "event.end": "2019-08-26T19:48:53.000Z", + "event.kind": "event", + "event.module": "aws", + "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA", + "event.start": "2019-08-26T19:47:55.000Z", + "event.type": "flow", + "fileset.name": "vpcflow", + "input.type": "log", + "log.offset": 526, + "service.type": "aws", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file From 4daebbd404e58a0c2be96e9f1fc3f8089c67b4db Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 23 Nov 2020 15:19:20 -0700 Subject: [PATCH 2/5] add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a2453a39a691..5ad93be09a3c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -332,6 +332,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix handing missing eventtime and assignip field being set to N/A for fortinet module. {pull}22361[22361] - Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696] - Fix for `field [source] not present as part of path [source.ip]` error in azure pipelines. {pull}22377[22377] +- Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716] *Heartbeat* From 743891e3bfa08b3c989ab035a785f020ef2e253e Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 25 Nov 2020 20:59:46 -0700 Subject: [PATCH 3/5] Add painless script to remove all empty fields --- .../module/aws/vpcflow/config/input.yml | 8 ------- .../module/aws/vpcflow/ingest/pipeline.yml | 24 +++++++++++++++++++ .../accept-reject-traffic.log-expected.json | 4 ---- .../test/custom-nat-gateway.log-expected.json | 1 - .../test/no-data-skip-data.log-expected.json | 2 -- ...-flag-sequence-skip-data.log-expected.json | 6 ----- .../test/tcp-flag-sequence.log-expected.json | 1 - 7 files changed, 24 insertions(+), 22 deletions(-) diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml index 43ec062acfe2..de4affbd694c 100644 --- a/x-pack/filebeat/module/aws/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/aws/vpcflow/config/input.yml @@ -129,14 +129,6 @@ processors: - drop_fields: fields: ["aws.vpcflow.srcaddr", "aws.vpcflow.srcport", "aws.vpcflow.dstaddr", "aws.vpcflow.dstport", "aws.vpcflow.bytes", "aws.vpcflow.packets", "aws.vpcflow.protocol"] - - drop_fields: - when.equals.aws.vpcflow.pkt_srcaddr: "-" - fields: ["aws.vpcflow.pkt_srcaddr"] - - - drop_fields: - when.equals.aws.vpcflow.pkt_dstaddr: "-" - fields: [ "aws.vpcflow.pkt_dstaddr" ] - - community_id: ~ # Use the aws.vpcflow.action value to set the event.outcome value to either "allow" or "deny". diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml index bd9b1d32769b..33f6d83047f5 100644 --- a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml @@ -28,6 +28,30 @@ processors: field: ["aws.vpcflow.start", "aws.vpcflow.end"] ignore_missing: true + - script: + lang: painless + source: >- + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v instanceof String && v == "-"); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx.aws); + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/aws/vpcflow/test/accept-reject-traffic.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/accept-reject-traffic.log-expected.json index 1f1b3e061b24..170b8851ec91 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/accept-reject-traffic.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/accept-reject-traffic.log-expected.json @@ -13,7 +13,6 @@ "destination.as.organization.name": "Consorci de Serveis Universitaris de Catalunya", "destination.geo.continent_name": "Europe", "destination.geo.country_iso_code": "ES", - "destination.geo.country_name": "Spain", "destination.geo.location.lat": 40.4172, "destination.geo.location.lon": -3.684, "destination.ip": "158.109.0.1", @@ -48,7 +47,6 @@ "source.geo.city_name": "Moscow", "source.geo.continent_name": "Europe", "source.geo.country_iso_code": "RU", - "source.geo.country_name": "Russia", "source.geo.location.lat": 55.7527, "source.geo.location.lon": 37.6172, "source.geo.region_iso_code": "RU-MOW", @@ -74,7 +72,6 @@ "destination.as.organization.name": "Consorci de Serveis Universitaris de Catalunya", "destination.geo.continent_name": "Europe", "destination.geo.country_iso_code": "ES", - "destination.geo.country_name": "Spain", "destination.geo.location.lat": 40.4172, "destination.geo.location.lon": -3.684, "destination.ip": "158.109.0.1", @@ -109,7 +106,6 @@ "source.geo.city_name": "Moscow", "source.geo.continent_name": "Europe", "source.geo.country_iso_code": "RU", - "source.geo.country_name": "Russia", "source.geo.location.lat": 55.7527, "source.geo.location.lon": 37.6172, "source.geo.region_iso_code": "RU-MOW", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json index d508bd634792..6b9e4382bb50 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json @@ -1,6 +1,5 @@ [ { - "aws.vpcflow.instance_id": "-", "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.pkt_dstaddr": "203.0.113.5", "aws.vpcflow.pkt_srcaddr": "10.0.1.5", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json index 22705d87101b..e8224ee08b11 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json @@ -2,7 +2,6 @@ { "@timestamp": "2015-05-10T18:02:14.000Z", "aws.vpcflow.account_id": "123456789010", - "aws.vpcflow.action": "-", "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.log_status": "NODATA", "aws.vpcflow.version": "2", @@ -27,7 +26,6 @@ { "@timestamp": "2015-05-10T18:02:14.000Z", "aws.vpcflow.account_id": "123456789010", - "aws.vpcflow.action": "-", "aws.vpcflow.interface_id": "eni-11111111aaaaaaaaa", "aws.vpcflow.log_status": "SKIPDATA", "aws.vpcflow.version": "2", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json index aa9cc75cfe73..b28207021b6a 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json @@ -2,13 +2,10 @@ { "@timestamp": "2019-08-26T19:48:53.000Z", "aws.vpcflow.account_id": "123456789010", - "aws.vpcflow.action": "-", "aws.vpcflow.instance_id": "i-01234567890123456", "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.log_status": "SKIPDATA", "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", - "aws.vpcflow.tcp_flags": "-", - "aws.vpcflow.type": "-", "aws.vpcflow.version": "3", "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", "cloud.account.id": "123456789010", @@ -33,13 +30,10 @@ { "@timestamp": "2019-08-26T19:48:53.000Z", "aws.vpcflow.account_id": "123456789010", - "aws.vpcflow.action": "-", "aws.vpcflow.instance_id": "i-01234567890123456", "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.log_status": "NODATA", "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", - "aws.vpcflow.tcp_flags": "-", - "aws.vpcflow.type": "-", "aws.vpcflow.version": "3", "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", "cloud.account.id": "123456789010", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json index ba0293752ca0..6b7b788ac972 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json @@ -49,7 +49,6 @@ "source.geo.city_name": "Dublin", "source.geo.continent_name": "Europe", "source.geo.country_iso_code": "IE", - "source.geo.country_name": "Ireland", "source.geo.location.lat": 53.3338, "source.geo.location.lon": -6.2488, "source.geo.region_iso_code": "IE-L", From bb35461c3e7e7f1e650121a3872129eea766359a Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Sun, 29 Nov 2020 09:56:32 -0700 Subject: [PATCH 4/5] update expected.json files --- .../aws/vpcflow/test/accept-reject-traffic.log-expected.json | 4 ++++ .../aws/vpcflow/test/tcp-flag-sequence.log-expected.json | 1 + 2 files changed, 5 insertions(+) diff --git a/x-pack/filebeat/module/aws/vpcflow/test/accept-reject-traffic.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/accept-reject-traffic.log-expected.json index 170b8851ec91..1f1b3e061b24 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/accept-reject-traffic.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/accept-reject-traffic.log-expected.json @@ -13,6 +13,7 @@ "destination.as.organization.name": "Consorci de Serveis Universitaris de Catalunya", "destination.geo.continent_name": "Europe", "destination.geo.country_iso_code": "ES", + "destination.geo.country_name": "Spain", "destination.geo.location.lat": 40.4172, "destination.geo.location.lon": -3.684, "destination.ip": "158.109.0.1", @@ -47,6 +48,7 @@ "source.geo.city_name": "Moscow", "source.geo.continent_name": "Europe", "source.geo.country_iso_code": "RU", + "source.geo.country_name": "Russia", "source.geo.location.lat": 55.7527, "source.geo.location.lon": 37.6172, "source.geo.region_iso_code": "RU-MOW", @@ -72,6 +74,7 @@ "destination.as.organization.name": "Consorci de Serveis Universitaris de Catalunya", "destination.geo.continent_name": "Europe", "destination.geo.country_iso_code": "ES", + "destination.geo.country_name": "Spain", "destination.geo.location.lat": 40.4172, "destination.geo.location.lon": -3.684, "destination.ip": "158.109.0.1", @@ -106,6 +109,7 @@ "source.geo.city_name": "Moscow", "source.geo.continent_name": "Europe", "source.geo.country_iso_code": "RU", + "source.geo.country_name": "Russia", "source.geo.location.lat": 55.7527, "source.geo.location.lon": 37.6172, "source.geo.region_iso_code": "RU-MOW", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json index 6b7b788ac972..ba0293752ca0 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json @@ -49,6 +49,7 @@ "source.geo.city_name": "Dublin", "source.geo.continent_name": "Europe", "source.geo.country_iso_code": "IE", + "source.geo.country_name": "Ireland", "source.geo.location.lat": 53.3338, "source.geo.location.lon": -6.2488, "source.geo.region_iso_code": "IE-L", From a1c222c9f57fccb2193c29e12016e868990ff9ed Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Sun, 29 Nov 2020 10:10:57 -0700 Subject: [PATCH 5/5] add ignore_failure: true --- x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml index 33f6d83047f5..a8a6e5ae7262 100644 --- a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml @@ -30,6 +30,8 @@ processors: - script: lang: painless + ignore_failure: true + if: ctx?.aws != null source: >- void handleMap(Map map) { for (def x : map.values()) {