From 59af914564496cad77d041748245b73535649c21 Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Wed, 28 Apr 2021 09:29:18 +0200 Subject: [PATCH 1/9] Add setOrchestratorMetadata to populate orchestrator.* fields --- .../module/gcp/audit/config/pipeline.js | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/x-pack/filebeat/module/gcp/audit/config/pipeline.js b/x-pack/filebeat/module/gcp/audit/config/pipeline.js index 878f2b19b8dd..2015852a159a 100644 --- a/x-pack/filebeat/module/gcp/audit/config/pipeline.js +++ b/x-pack/filebeat/module/gcp/audit/config/pipeline.js @@ -63,6 +63,35 @@ function Audit(keep_original_message) { fail_on_error: false, }); + var setOrchestratorMetadata = function(evt) { + if (evt.Get("json.resource.type") === "k8s_cluster") { + evt.Put("orchestrator.type", "kubernetes"); + // Dissect to extract the api_version + var dissect_processor = new processor.Dissect({ + "tokenizer": "%{}/%{orchestrator.api_version}/%{}", + "field": "json.protoPayload.resourceName", + "target_prefix": "", + }).Run; + + var convert_processor = new processor.Convert({ + fields: [ + { + from: "json.resource.labels.cluster_name", + to: "orchestrator.cluster.name", + type: "string" + }, + { + from: "json.protoPayload.resourceName", + to: "orchestrator.resource.type", + type: "string" + } + ], + ignore_missing: true, + fail_on_error: false, + }).Run; + } + }; + // The log includes a protoPayload field. // https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry var convertLogEntry = new processor.Convert({ @@ -290,6 +319,7 @@ function Audit(keep_original_message) { .Add(dropPubSubFields) .Add(saveMetadata) .Add(setCloudMetadata) + .Add(setOrchestratorMetadata) .Add(convertLogEntry) .Add(convertProtoPayload) .Add(copyFields) From 778f40fc6eeaaf3a506d9ac7cbc4d0b6fc1b7c6f Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Wed, 28 Apr 2021 09:43:28 +0200 Subject: [PATCH 2/9] Correct setOrchestratorMetadata --- x-pack/filebeat/module/gcp/audit/config/pipeline.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/module/gcp/audit/config/pipeline.js b/x-pack/filebeat/module/gcp/audit/config/pipeline.js index 2015852a159a..c73af4d63eaf 100644 --- a/x-pack/filebeat/module/gcp/audit/config/pipeline.js +++ b/x-pack/filebeat/module/gcp/audit/config/pipeline.js @@ -71,7 +71,7 @@ function Audit(keep_original_message) { "tokenizer": "%{}/%{orchestrator.api_version}/%{}", "field": "json.protoPayload.resourceName", "target_prefix": "", - }).Run; + }).Run(evt); var convert_processor = new processor.Convert({ fields: [ @@ -88,7 +88,7 @@ function Audit(keep_original_message) { ], ignore_missing: true, fail_on_error: false, - }).Run; + }).Run(evt); } }; From 912ef55c08341848397e1a88825dbdc120550f0f Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Thu, 29 Apr 2021 15:39:08 +0200 Subject: [PATCH 3/9] Improve parsing to populate orchestrator.* fields Add sample logs --- .../filebeat/module/gcp/audit/config/pipeline.js | 9 +-------- .../filebeat/module/gcp/audit/ingest/pipeline.yml | 15 +++++++++++++++ .../gcp/audit/test/audit-log-entries.json.log | 4 ++++ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/x-pack/filebeat/module/gcp/audit/config/pipeline.js b/x-pack/filebeat/module/gcp/audit/config/pipeline.js index c73af4d63eaf..8c06bcec37fb 100644 --- a/x-pack/filebeat/module/gcp/audit/config/pipeline.js +++ b/x-pack/filebeat/module/gcp/audit/config/pipeline.js @@ -66,13 +66,6 @@ function Audit(keep_original_message) { var setOrchestratorMetadata = function(evt) { if (evt.Get("json.resource.type") === "k8s_cluster") { evt.Put("orchestrator.type", "kubernetes"); - // Dissect to extract the api_version - var dissect_processor = new processor.Dissect({ - "tokenizer": "%{}/%{orchestrator.api_version}/%{}", - "field": "json.protoPayload.resourceName", - "target_prefix": "", - }).Run(evt); - var convert_processor = new processor.Convert({ fields: [ { @@ -82,7 +75,7 @@ function Audit(keep_original_message) { }, { from: "json.protoPayload.resourceName", - to: "orchestrator.resource.type", + to: "orchestrator.resource.type_temp", type: "string" } ], diff --git a/x-pack/filebeat/module/gcp/audit/ingest/pipeline.yml b/x-pack/filebeat/module/gcp/audit/ingest/pipeline.yml index 8e0d3ac6fdb7..c01fe781e050 100644 --- a/x-pack/filebeat/module/gcp/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/gcp/audit/ingest/pipeline.yml @@ -29,6 +29,21 @@ processors: field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true + - grok: + field: orchestrator.resource.type_temp + patterns: + - '%{DATA}/%{API_VERSION:orchestrator.api_version}/namespaces/%{DATA:orchestrator.namespace}/%{RESOURCE_TYPE:orchestrator.resource.type}(/%{HOSTNAME:orchestrator.resource.name})?' + - '%{DATA}/%{API_VERSION:orchestrator.api_version}/%{RESOURCE_TYPE:orchestrator.resource.type}' + - 'apis/%{RESOURCE_TYPE:orchestrator.resource.type}/%{API_VERSION:orchestrator.api_version}' + - 'api/%{API_VERSION:orchestrator.api_version}' + - '%{RESOURCE_TYPE:orchestrator.resource.type}' + pattern_definitions: + API_VERSION: (v\d+([a-z]+)?(\d+)?) + RESOURCE_TYPE: ([a-z]+((\.[a-z0-9]+)+)?) + ignore_missing: true + - remove: + field: orchestrator.resource.type_temp + ignore_missing: true on_failure: - set: diff --git a/x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log b/x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log index 9c2288905273..df986980c71b 100644 --- a/x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log +++ b/x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log @@ -5,3 +5,7 @@ {"insertId":"87efd529-6349-45d2-b905-fc607e6c5d3b","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cert-manager-webhook:auth-delegator\" of ClusterRole \"system:auth-delegator\" to ServiceAccount \"cert-manager-webhook/cert-manager\""},"logName":"projects/foo/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"5555555-6349-45d2-b905-fc607e6c5d3b","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:serviceaccount:cert-manager:cert-manager-webhook"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","resource":"authorization.k8s.io/v1beta1/subjectaccessreviews"}],"methodName":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","request":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":false}},"requestMetadata":{"callerIp":"10.11.12.13","callerSuppliedUserAgent":"webhook/v0.0.0 (linux/amd64) kubernetes/$Format"},"resourceName":"authorization.k8s.io/v1beta1/subjectaccessreviews","response":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":true,"reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""}},"serviceName":"k8s.io","status":{"code":0}},"receiveTimestamp":"2020-08-05T21:07:32.157698684Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2020-08-05T21:07:30.974750Z"} {"insertId":"v2spcwdzmc2","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"first":true,"id":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"user@mycompany.com"},"authorizationInfo":[{"granted":true,"permission":"compute.images.create","resourceAttributes":{"name":"projects/foo/global/images/windows-server-2016-v20200805","service":"compute","type":"compute.images"}}],"methodName":"v1.compute.images.insert","request":{"@type":"type.googleapis.com/compute.images.insert","family":"windows-server-2016","guestOsFeatures":[{"type":"VIRTIO_SCSI_MULTIQUEUE"},{"type":"WINDOWS"}],"name":"windows-server-2016-v20200805","rawDisk":{"source":"https://storage.googleapis.com/storage/v1/b/foo/o/windows-server-2016-v20200805.tar.gz"},"sourceType":"RAW"},"requestMetadata":{"callerIp":"1.2.3.4","callerSuppliedUserAgent":"google-cloud-sdk gcloud/290.0.1 command/gcloud.compute.images.create invocation-id/032752ad0fa44b4ea951951d2deef6a3 environment/None environment-version/None interactive/True from-script/False python/2.7.17 term/xterm-256color (Macintosh; Intel Mac OS X 19.6.0),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2020-08-05T21:59:27.515Z"}},"resourceLocation":{"currentLocations":["eu"]},"resourceName":"projects/foo/global/images/windows-server-2016-v20200805","response":{"@type":"type.googleapis.com/operation","id":"44919313","insertTime":"2020-08-05T14:59:27.259-07:00","name":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","operationType":"insert","progress":"0","selfLink":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/operation-1596664766354-5ac287c395484-fa3923bd-543e018e","selfLinkWithId":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/4491931805423146320","startTime":"2020-08-05T14:59:27.274-07:00","status":"RUNNING","targetId":"12345","targetLink":"https://www.googleapis.com/compute/v1/projects/foo/global/images/windows-server-2016-v20200805","user":"user@mycompany.com"},"serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T21:59:27.822546978Z","resource":{"labels":{"image_id":"771879043","project_id":"foo"},"type":"gce_image"},"severity":"NOTICE","timestamp":"2020-08-05T21:59:26.456Z"} {"insertId":"-c7ctxmd2zab","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"id":"operation-1596646123456-5ac2438b775f6-f8ca1382-e70b6831","last":true,"producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"user@mycompany.com"},"methodName":"beta.compute.instances.stop","request":{"@type":"type.googleapis.com/compute.instances.stop"},"requestMetadata":{"callerIp":"2.3.4.5","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:79.0) Gecko/20100101 Firefox/79.0,gzip(gfe),gzip(gfe)"},"resourceName":"projects/foo/zones/us-central1-a/instances/win10-test","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T16:56:41.315135528Z","resource":{"labels":{"instance_id":"590261181","project_id":"foo","zone":"us-central1-a"},"type":"gce_instance"},"severity":"NOTICE","timestamp":"2020-08-05T16:56:40.428Z"} +{"insertId":"94170ac4-6e82-4345-98ad-3c780222d19d","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":""},"logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"94170ac4-6e82-4345-98ad-3c780222d19d","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.core.v1.nodes.list","resource":"core/v1/nodes"}],"methodName":"io.k8s.core.v1.nodes.list","requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"GoogleCloudConsole"},"resourceName":"core/v1/nodes","serviceName":"k8s.io","status":{}},"receiveTimestamp":"2021-04-23T14:47:31.94822935Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2021-04-23T14:47:07.535383Z"} +{"insertId":"b10a904a-faa4-4e0d-9ec3-7bc6a180196a","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"","k8s.io/deprecated":"true","k8s.io/removed-release":"1.22"},"logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"b10a904a-faa4-4e0d-9ec3-7bc6a180196a","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.extensions.v1beta1.ingresses.list","resource":"extensions/v1beta1/namespaces/cos-auditd/ingresses"}],"methodName":"io.k8s.extensions.v1beta1.ingresses.list","requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"GoogleCloudConsole"},"resourceName":"extensions/v1beta1/namespaces/cos-auditd/ingresses","serviceName":"k8s.io","status":{}},"receiveTimestamp":"2021-04-23T14:16:36.37362467Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2021-04-23T14:16:07.574776Z"} +{"insertId":"e973134d-b4d5-4e2f-92b8-82bba13fdb92","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:public-info-viewer\" of ClusterRole \"system:public-info-viewer\" to Group \"system:unauthenticated\""},"logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"e973134d-b4d5-4e2f-92b8-82bba13fdb92","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:anonymous"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.get","resource":"readyz"}],"methodName":"io.k8s.get","requestMetadata":{"callerIp":"127.0.0.1","callerSuppliedUserAgent":"kube-probe/1.19+"},"resourceName":"readyz","serviceName":"k8s.io","status":{}},"receiveTimestamp":"2021-04-29T08:19:21.606980385Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2021-04-29T08:19:20.80581Z"} +{"insertId":"03adfb9f-71a3-4f41-9701-29b5542f4d22","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""},"logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"03adfb9f-71a3-4f41-9701-29b5542f4d22","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:serviceaccount:kube-system:generic-garbage-collector"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.get","resource":"api/v1"}],"methodName":"io.k8s.get","requestMetadata":{"callerIp":"::1","callerSuppliedUserAgent":"kube-controller-manager/v1.19.8 (linux/amd64) kubernetes/4f6f69f/system:serviceaccount:kube-system:generic-garbage-collector"},"resourceName":"api/v1","serviceName":"k8s.io","status":{}},"receiveTimestamp":"2021-04-29T08:23:19.71757101Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2021-04-29T08:23:18.899153Z"} From e448521efc45af6e003a1875ab18cae4406069a9 Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Thu, 29 Apr 2021 15:53:48 +0200 Subject: [PATCH 4/9] add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 529b8361342c..a6fb96cf7c00 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -107,6 +107,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Possible values for Netflow's locality fields (source.locality, destination.locality and flow.locality) are now `internal` and `external`, instead of `private` and `public`. {issue}24272[24272] {pull}24295[24295] - Add User Agent Parser for Azure Sign In Logs Ingest Pipeline {pull}23201[23201] - Changes filebeat httpjson input's append transform to create a list even with only a single value{pull}25074[25074] +- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368] *Heartbeat* From 14ed1183f79c043fbe176d15f5032f2d97f5f8e7 Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 1 Jun 2021 12:35:40 +0200 Subject: [PATCH 5/9] Bump ECS version to 1.10 --- x-pack/filebeat/module/gcp/audit/config/input.yml | 2 +- x-pack/filebeat/module/gcp/firewall/config/input.yml | 2 +- x-pack/filebeat/module/gcp/vpcflow/config/input.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/filebeat/module/gcp/audit/config/input.yml b/x-pack/filebeat/module/gcp/audit/config/input.yml index 4945e01447b8..71eff9708e01 100644 --- a/x-pack/filebeat/module/gcp/audit/config/input.yml +++ b/x-pack/filebeat/module/gcp/audit/config/input.yml @@ -34,4 +34,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.9.0 + ecs.version: 1.10.0 diff --git a/x-pack/filebeat/module/gcp/firewall/config/input.yml b/x-pack/filebeat/module/gcp/firewall/config/input.yml index 05e4fc5c10e8..8f2d52cda389 100644 --- a/x-pack/filebeat/module/gcp/firewall/config/input.yml +++ b/x-pack/filebeat/module/gcp/firewall/config/input.yml @@ -38,4 +38,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.9.0 + ecs.version: 1.10.0 diff --git a/x-pack/filebeat/module/gcp/vpcflow/config/input.yml b/x-pack/filebeat/module/gcp/vpcflow/config/input.yml index ded34be1443d..9472eb97619d 100644 --- a/x-pack/filebeat/module/gcp/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/gcp/vpcflow/config/input.yml @@ -37,4 +37,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.9.0 + ecs.version: 1.10.0 From ecafd87f5c8290cd9959c36408a05ced916b368e Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 1 Jun 2021 12:55:25 +0200 Subject: [PATCH 6/9] Add generated data based on sample logs --- .../audit-log-entries.json.log-expected.json | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log-expected.json b/x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log-expected.json index 26abbf7ec804..a25ec4fb1f3a 100644 --- a/x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log-expected.json +++ b/x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log-expected.json @@ -227,6 +227,10 @@ "input.type": "log", "log.logger": "projects/foo/logs/cloudaudit.googleapis.com%2Fdata_access", "log.offset": 5100, + "orchestrator.api_version": "v1beta1", + "orchestrator.cluster.name": "analysis-cluster", + "orchestrator.resource.type": "subjectaccessreviews", + "orchestrator.type": "kubernetes", "service.name": "k8s.io", "service.type": "gcp", "source.ip": "10.11.12.13", @@ -345,5 +349,173 @@ "user_agent.os.name": "Mac OS X", "user_agent.os.version": "10.15", "user_agent.version": "79.0." + }, + { + "@timestamp": "2021-04-23T14:47:07.535Z", + "cloud.project.id": "elastic-siem", + "event.action": "io.k8s.core.v1.nodes.list", + "event.dataset": "gcp.audit", + "event.id": "94170ac4-6e82-4345-98ad-3c780222d19d", + "event.kind": "event", + "event.module": "gcp", + "event.outcome": "success", + "fileset.name": "audit", + "gcp.audit.authentication_info.principal_email": "xxx@xxx.xxx", + "gcp.audit.authorization_info": [ + { + "granted": true, + "permission": "io.k8s.core.v1.nodes.list", + "resource": "core/v1/nodes" + } + ], + "gcp.audit.method_name": "io.k8s.core.v1.nodes.list", + "gcp.audit.request_metadata.caller_ip": "192.168.1.1", + "gcp.audit.request_metadata.caller_supplied_user_agent": "GoogleCloudConsole", + "gcp.audit.resource_name": "core/v1/nodes", + "gcp.audit.service_name": "k8s.io", + "gcp.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog", + "input.type": "log", + "log.logger": "projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access", + "log.offset": 10917, + "orchestrator.api_version": "v1", + "orchestrator.cluster.name": "analysis-cluster", + "orchestrator.resource.type": "nodes", + "orchestrator.type": "kubernetes", + "service.name": "k8s.io", + "service.type": "gcp", + "source.ip": "192.168.1.1", + "tags": [ + "forwarded" + ], + "user.email": "xxx@xxx.xxx", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "GoogleCloudConsole" + }, + { + "@timestamp": "2021-04-23T14:16:07.574Z", + "cloud.project.id": "elastic-siem", + "event.action": "io.k8s.extensions.v1beta1.ingresses.list", + "event.dataset": "gcp.audit", + "event.id": "b10a904a-faa4-4e0d-9ec3-7bc6a180196a", + "event.kind": "event", + "event.module": "gcp", + "event.outcome": "success", + "fileset.name": "audit", + "gcp.audit.authentication_info.principal_email": "xxx@xxx.xxx", + "gcp.audit.authorization_info": [ + { + "granted": true, + "permission": "io.k8s.extensions.v1beta1.ingresses.list", + "resource": "extensions/v1beta1/namespaces/cos-auditd/ingresses" + } + ], + "gcp.audit.method_name": "io.k8s.extensions.v1beta1.ingresses.list", + "gcp.audit.request_metadata.caller_ip": "192.168.1.1", + "gcp.audit.request_metadata.caller_supplied_user_agent": "GoogleCloudConsole", + "gcp.audit.resource_name": "extensions/v1beta1/namespaces/cos-auditd/ingresses", + "gcp.audit.service_name": "k8s.io", + "gcp.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog", + "input.type": "log", + "log.logger": "projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access", + "log.offset": 11897, + "orchestrator.api_version": "v1beta1", + "orchestrator.cluster.name": "analysis-cluster", + "orchestrator.namespace": "cos-auditd", + "orchestrator.resource.type": "ingresses", + "orchestrator.type": "kubernetes", + "service.name": "k8s.io", + "service.type": "gcp", + "source.ip": "192.168.1.1", + "tags": [ + "forwarded" + ], + "user.email": "xxx@xxx.xxx", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "GoogleCloudConsole" + }, + { + "@timestamp": "2021-04-29T08:19:20.805Z", + "cloud.project.id": "elastic-siem", + "event.action": "io.k8s.get", + "event.dataset": "gcp.audit", + "event.id": "e973134d-b4d5-4e2f-92b8-82bba13fdb92", + "event.kind": "event", + "event.module": "gcp", + "event.outcome": "success", + "fileset.name": "audit", + "gcp.audit.authentication_info.principal_email": "system:anonymous", + "gcp.audit.authorization_info": [ + { + "granted": true, + "permission": "io.k8s.get", + "resource": "readyz" + } + ], + "gcp.audit.method_name": "io.k8s.get", + "gcp.audit.request_metadata.caller_ip": "127.0.0.1", + "gcp.audit.request_metadata.caller_supplied_user_agent": "kube-probe/1.19+", + "gcp.audit.resource_name": "readyz", + "gcp.audit.service_name": "k8s.io", + "gcp.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog", + "input.type": "log", + "log.logger": "projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access", + "log.offset": 13040, + "orchestrator.cluster.name": "analysis-cluster", + "orchestrator.resource.type": "readyz", + "orchestrator.type": "kubernetes", + "service.name": "k8s.io", + "service.type": "gcp", + "source.ip": "127.0.0.1", + "tags": [ + "forwarded" + ], + "user.email": "system:anonymous", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "kube-probe/1.19+" + }, + { + "@timestamp": "2021-04-29T08:23:18.899Z", + "cloud.project.id": "elastic-siem", + "event.action": "io.k8s.get", + "event.dataset": "gcp.audit", + "event.id": "03adfb9f-71a3-4f41-9701-29b5542f4d22", + "event.kind": "event", + "event.module": "gcp", + "event.outcome": "success", + "fileset.name": "audit", + "gcp.audit.authentication_info.principal_email": "system:serviceaccount:kube-system:generic-garbage-collector", + "gcp.audit.authorization_info": [ + { + "granted": true, + "permission": "io.k8s.get", + "resource": "api/v1" + } + ], + "gcp.audit.method_name": "io.k8s.get", + "gcp.audit.request_metadata.caller_ip": "::1", + "gcp.audit.request_metadata.caller_supplied_user_agent": "kube-controller-manager/v1.19.8 (linux/amd64) kubernetes/4f6f69f/system:serviceaccount:kube-system:generic-garbage-collector", + "gcp.audit.resource_name": "api/v1", + "gcp.audit.service_name": "k8s.io", + "gcp.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog", + "input.type": "log", + "log.logger": "projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access", + "log.offset": 14123, + "orchestrator.api_version": "v1", + "orchestrator.cluster.name": "analysis-cluster", + "orchestrator.type": "kubernetes", + "service.name": "k8s.io", + "service.type": "gcp", + "source.ip": "::1", + "tags": [ + "forwarded" + ], + "user.email": "system:serviceaccount:kube-system:generic-garbage-collector", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "kube-controller-manager/v1.19.8 (linux/amd64) kubernetes/4f6f69f/system:serviceaccount:kube-system:generic-garbage-collector", + "user_agent.os.name": "Linux" } ] \ No newline at end of file From ceeee698514d1293ef0317f1f452dcc004ccc762 Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 1 Jun 2021 13:36:31 +0200 Subject: [PATCH 7/9] Correct CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 271c6af6f4b1..36d53061ecd8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -103,10 +103,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename bad ECS field name tracing.trace.id to trace.id in aws elb fileset. {pull}22571[22571] - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975] - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041] -- Add User Agent Parser for Azure Sign In Logs Ingest Pipeline {pull}23201[23201] -- Changes filebeat httpjson input's append transform to create a list even with only a single value{pull}25074[25074] -- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368] - Change logging in logs input to structure logging. Some log message formats have changed. {pull}25299[25299] +- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368] *Heartbeat* From 09e5aa0c041a5ba6a008b589694f0599c432576e Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 1 Jun 2021 13:44:59 +0200 Subject: [PATCH 8/9] Correct CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 36d53061ecd8..e3d202a33b20 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -104,7 +104,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975] - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041] - Change logging in logs input to structure logging. Some log message formats have changed. {pull}25299[25299] -- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368] + *Heartbeat* @@ -810,6 +810,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - In Cisco Umbrella fileset add users from cisco.umbrella.identities to related.user. {pull}25776[25776] - Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841] - Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686] +- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368] *Heartbeat* From 9b88e3db31ef5e5ccfd5c758503afe0bfc2eeb66 Mon Sep 17 00:00:00 2001 From: Antoine Ryon Date: Tue, 1 Jun 2021 13:52:21 +0200 Subject: [PATCH 9/9] Correct CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e3d202a33b20..0eb9b4867c87 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -105,7 +105,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041] - Change logging in logs input to structure logging. Some log message formats have changed. {pull}25299[25299] - *Heartbeat* *Journalbeat*