From 66a05e5ed8fc85534e68cba0bcc0d0305fed6f7f Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Wed, 14 Apr 2021 17:33:48 +0200 Subject: [PATCH] [Filebeat][httpjson] Change append transform to initiate new fields as a slice (#25074) * changing the append processor to create a single value as list, not only if two or more values are added * removing mistake typo * add changelog entry (cherry picked from commit 2b11cf81c9eb26e18afef557ce5c6679a558c369) --- CHANGELOG.next.asciidoc | 3 +++ x-pack/filebeat/docs/inputs/input-httpjson.asciidoc | 2 +- .../input/httpjson/internal/v2/transform_append.go | 4 +++- .../httpjson/internal/v2/transform_append_test.go | 10 ++++++++++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index eb9c724f93a8..6af6c742f4d5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -47,6 +47,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975] - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041] - Rename `s3` input to `aws-s3` input. {pull}23469[23469] +- Possible values for Netflow's locality fields (source.locality, destination.locality and flow.locality) are now `internal` and `external`, instead of `private` and `public`. {issue}24272[24272] {pull}24295[24295] +- Add User Agent Parser for Azure Sign In Logs Ingest Pipeline {pull}23201[23201] +- Changes filebeat httpjson input's append transform to create a list even with only a single value{pull}25074[25074] *Heartbeat* diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 64247b517f82..8fedbcebeeb5 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -120,7 +120,7 @@ The access limitations are described in the corresponding configuration sections [float] ==== `append` -Appends a value to a list. If the field does not exist, the first entry will be a scalar value, and subsequent additions will convert the value to a list. +Appends a value to an array. If the field does not exist, the first entry will create a new array. If the field exists, the value is appended to the existing field and converted to a list. ["source","yaml",subs="attributes"] ---- diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go index 6a5867e5bbb9..4418892fb6c2 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go @@ -119,7 +119,7 @@ func appendToCommonMap(m common.MapStr, key, val string) error { if val == "" { return nil } - var value interface{} = val + var value interface{} if found, _ := m.HasKey(key); found { prev, _ := m.GetValue(key) switch t := prev.(type) { @@ -131,6 +131,8 @@ func appendToCommonMap(m common.MapStr, key, val string) error { value = []interface{}{prev, val} } + } else { + value = []interface{}{val} } if _, err := m.Put(key, value); err != nil { return err diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go index 12b49af4395f..73cb3e780269 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go @@ -147,6 +147,16 @@ func TestAppendFunctions(t *testing.T) { expectedTr: transformable{"body": common.MapStr{"a_key": []interface{}{"a_value", "another_value"}}}, expectedErr: nil, }, + { + name: "appendBodyWithSingleValue", + tfunc: appendBody, + paramCtx: &transformContext{}, + paramTr: transformable{"body": common.MapStr{}}, + paramKey: "a_key", + paramVal: "a_value", + expectedTr: transformable{"body": common.MapStr{"a_key": []interface{}{"a_value"}}}, + expectedErr: nil, + }, { name: "appendHeader", tfunc: appendHeader,