From 0347762b9b122a17e0f05f5fa3f56563ac86e6f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 3 Feb 2020 15:25:19 +0100 Subject: [PATCH] Make field names in Functionbeat ECS compliant (#16000) This PR fixes an issue in the `storage` function of Functionbeat. ES was not able to index events coming from the function as one of its fields had a different type: ``` { "textPayload": "2020-01-31T15:03:10.758Z\tWARN\telasticsearch/client.go:516\tCannot index event publisher.Event{Content:beat.Event{Timestamp:time.Time{wall:0xf60c480, ext:63716079787, loc:(*time.Location)(nil)}, Meta:null, Fields:{\"agent\":{\"ephemeral_id\":\"d86f3440-cb23-431a-81b7-d3360017ad2b\",\"hostname\":\"localhost\",\"id\":\"1a0a7af8-b927-460f-9746-7f8eae8dd417\",\"type\":\"functionbeat\",\"version\":\"7.6.0\"},\"bucket\":\"test-elastic-bucket\",\"created\":\"2020-01-31T15:03:06.783Z\",\"ecs\":{\"version\":\"1.4.0\"},\"file\":\"Untitled\",\"host\":{\"name\":\"localhost\"},\"id\":\"951771144251331\",\"metageneration\":\"1\",\"read_timestamp\":\"2020-01-31T15:03:10.209Z\",\"resource\":{\"event_type\":\"storage#object\",\"name\":\"projects/_/buckets/test-elastic-bucket/objects/Untitled\",\"service\":\"storage.googleapis.com\",\"state\":\"\"},\"updated\":\"2020-01-31T15:03:06.783Z\"}, Private:interface {}(nil), TimeSeries:false}, Flags:0x1, Cache:publisher.EventCache{m:common.MapStr(nil)}} (status=400): {\"type\":\"mapper_parsing_exception\",\"reason\":\"object mapping for [file] tried to parse field [file] as object, but found a concrete value\"}", "insertId": "000000-4f927011-4af5-4876-b471-62889ff0796b", "resource": { "type": "cloud_function", "labels": { "project_id": "elastic-metricbeat", "region": "europe-west2", "function_name": "storage" } }, "timestamp": "2020-01-31T15:03:10.758Z", "labels": { "execution_id": "951771144251331" }, "logName": "projects/elastic-metricbeat/logs/cloudfunctions.googleapis.com%2Fcloud-functions", "trace": "projects/elastic-metricbeat/traces/571a3f3523594e819691f0672c32c413", "receiveTimestamp": "2020-01-31T15:03:12.107405704Z" } ``` This problem prevented ES from indexing new storage events. --- x-pack/functionbeat/magefile.go | 5 +---- x-pack/functionbeat/provider/gcp/gcp/transformer.go | 13 ++++++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/functionbeat/magefile.go b/x-pack/functionbeat/magefile.go index 89e497d0f4f1..44e9765e397d 100644 --- a/x-pack/functionbeat/magefile.go +++ b/x-pack/functionbeat/magefile.go @@ -161,10 +161,7 @@ func GoTestUnit() { func BuildPkgForFunctions() error { mg.Deps(Update, Build) - err := os.MkdirAll("pkg", 700) - if err != nil { - return err - } + err := os.RemoveAll("pkg") filesToCopy := map[string]string{ filepath.Join("provider", "aws", "functionbeat-aws"): filepath.Join("pkg", "functionbeat-aws"), diff --git a/x-pack/functionbeat/provider/gcp/gcp/transformer.go b/x-pack/functionbeat/provider/gcp/gcp/transformer.go index 7518414ec206..9072597de514 100644 --- a/x-pack/functionbeat/provider/gcp/gcp/transformer.go +++ b/x-pack/functionbeat/provider/gcp/gcp/transformer.go @@ -56,11 +56,14 @@ func transformStorage(mData *metadata.Metadata, evt StorageEvent) (beat.Event, e "event_type": mData.Resource.Type, "state": evt.ResourceState, }, - "bucket": evt.Bucket, - "file": evt.Name, - "metageneration": evt.Metageneration, - "updated": evt.Updated, - "created": evt.Created, + "storage_bucket": evt.Bucket, + "file": common.MapStr{ + "name": evt.Name, + "mtime": evt.Updated, + "ctime": evt.Updated, + "created": evt.Created, + }, + "meta-generation": evt.Metageneration, }, }, nil }