[FileBeat] GCP module enhancement - Populate orchestrator.* fields for K8S logs

CHANGELOG.next.asciidoc

@@ -103,6 +103,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Rename bad ECS field name tracing.trace.id to trace.id in aws elb fileset. {pull}22571[22571]
 - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975]
 - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041]
+- Add User Agent Parser for Azure Sign In Logs Ingest Pipeline {pull}23201[23201]
+- Changes filebeat httpjson input's append transform to create a list even with only a single value{pull}25074[25074]
+- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368]
 - Change logging in logs input to structure logging. Some log message formats have changed. {pull}25299[25299]
 
 *Heartbeat*

x-pack/filebeat/module/gcp/audit/config/input.yml

@@ -34,4 +34,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.9.0
+        ecs.version: 1.10.0

x-pack/filebeat/module/gcp/audit/config/pipeline.js

@@ -63,6 +63,28 @@ function Audit(keep_original_message) {
         fail_on_error: false,
     });
 
+    var setOrchestratorMetadata = function(evt) {
+          if (evt.Get("json.resource.type") === "k8s_cluster") {
+            evt.Put("orchestrator.type", "kubernetes");
+            var convert_processor = new processor.Convert({
+                fields: [
+                    {
+                        from: "json.resource.labels.cluster_name",
+                        to: "orchestrator.cluster.name",
+                        type: "string"
+                    },
+                    {
+                        from: "json.protoPayload.resourceName",
+                        to: "orchestrator.resource.type_temp",
+                        type: "string"
+                    }
+                ],
+                ignore_missing: true,
+                fail_on_error: false,
+            }).Run(evt);
+        }
+    };
+
     // The log includes a protoPayload field.
     // https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry
     var convertLogEntry = new processor.Convert({
@@ -290,6 +312,7 @@ function Audit(keep_original_message) {
         .Add(dropPubSubFields)
         .Add(saveMetadata)
         .Add(setCloudMetadata)
+        .Add(setOrchestratorMetadata)
         .Add(convertLogEntry)
         .Add(convertProtoPayload)
         .Add(copyFields)

x-pack/filebeat/module/gcp/audit/ingest/pipeline.yml

@@ -29,6 +29,21 @@ processors:
       field: source.as.organization_name
       target_field: source.as.organization.name
       ignore_missing: true
+  - grok:
+      field: orchestrator.resource.type_temp
+      patterns:
+        - '%{DATA}/%{API_VERSION:orchestrator.api_version}/namespaces/%{DATA:orchestrator.namespace}/%{RESOURCE_TYPE:orchestrator.resource.type}(/%{HOSTNAME:orchestrator.resource.name})?'
+        - '%{DATA}/%{API_VERSION:orchestrator.api_version}/%{RESOURCE_TYPE:orchestrator.resource.type}'
+        - 'apis/%{RESOURCE_TYPE:orchestrator.resource.type}/%{API_VERSION:orchestrator.api_version}'
+        - 'api/%{API_VERSION:orchestrator.api_version}'
+        - '%{RESOURCE_TYPE:orchestrator.resource.type}'
+      pattern_definitions:
+        API_VERSION: (v\d+([a-z]+)?(\d+)?)
+        RESOURCE_TYPE: ([a-z]+((\.[a-z0-9]+)+)?)
+      ignore_missing: true
+  - remove:
+      field: orchestrator.resource.type_temp
+      ignore_missing: true
 
 on_failure:
   - set:

x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log

@@ -5,3 +5,7 @@
 {"insertId":"87efd529-6349-45d2-b905-fc607e6c5d3b","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cert-manager-webhook:auth-delegator\" of ClusterRole \"system:auth-delegator\" to ServiceAccount \"cert-manager-webhook/cert-manager\""},"logName":"projects/foo/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"5555555-6349-45d2-b905-fc607e6c5d3b","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:serviceaccount:cert-manager:cert-manager-webhook"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","resource":"authorization.k8s.io/v1beta1/subjectaccessreviews"}],"methodName":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","request":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":false}},"requestMetadata":{"callerIp":"10.11.12.13","callerSuppliedUserAgent":"webhook/v0.0.0 (linux/amd64) kubernetes/$Format"},"resourceName":"authorization.k8s.io/v1beta1/subjectaccessreviews","response":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":true,"reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""}},"serviceName":"k8s.io","status":{"code":0}},"receiveTimestamp":"2020-08-05T21:07:32.157698684Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2020-08-05T21:07:30.974750Z"}
 {"insertId":"v2spcwdzmc2","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"first":true,"id":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"user@mycompany.com"},"authorizationInfo":[{"granted":true,"permission":"compute.images.create","resourceAttributes":{"name":"projects/foo/global/images/windows-server-2016-v20200805","service":"compute","type":"compute.images"}}],"methodName":"v1.compute.images.insert","request":{"@type":"type.googleapis.com/compute.images.insert","family":"windows-server-2016","guestOsFeatures":[{"type":"VIRTIO_SCSI_MULTIQUEUE"},{"type":"WINDOWS"}],"name":"windows-server-2016-v20200805","rawDisk":{"source":"https://storage.googleapis.com/storage/v1/b/foo/o/windows-server-2016-v20200805.tar.gz"},"sourceType":"RAW"},"requestMetadata":{"callerIp":"1.2.3.4","callerSuppliedUserAgent":"google-cloud-sdk gcloud/290.0.1 command/gcloud.compute.images.create invocation-id/032752ad0fa44b4ea951951d2deef6a3 environment/None environment-version/None interactive/True from-script/False python/2.7.17 term/xterm-256color (Macintosh; Intel Mac OS X 19.6.0),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2020-08-05T21:59:27.515Z"}},"resourceLocation":{"currentLocations":["eu"]},"resourceName":"projects/foo/global/images/windows-server-2016-v20200805","response":{"@type":"type.googleapis.com/operation","id":"44919313","insertTime":"2020-08-05T14:59:27.259-07:00","name":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","operationType":"insert","progress":"0","selfLink":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/operation-1596664766354-5ac287c395484-fa3923bd-543e018e","selfLinkWithId":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/4491931805423146320","startTime":"2020-08-05T14:59:27.274-07:00","status":"RUNNING","targetId":"12345","targetLink":"https://www.googleapis.com/compute/v1/projects/foo/global/images/windows-server-2016-v20200805","user":"user@mycompany.com"},"serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T21:59:27.822546978Z","resource":{"labels":{"image_id":"771879043","project_id":"foo"},"type":"gce_image"},"severity":"NOTICE","timestamp":"2020-08-05T21:59:26.456Z"}
 {"insertId":"-c7ctxmd2zab","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"id":"operation-1596646123456-5ac2438b775f6-f8ca1382-e70b6831","last":true,"producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"user@mycompany.com"},"methodName":"beta.compute.instances.stop","request":{"@type":"type.googleapis.com/compute.instances.stop"},"requestMetadata":{"callerIp":"2.3.4.5","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:79.0) Gecko/20100101 Firefox/79.0,gzip(gfe),gzip(gfe)"},"resourceName":"projects/foo/zones/us-central1-a/instances/win10-test","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T16:56:41.315135528Z","resource":{"labels":{"instance_id":"590261181","project_id":"foo","zone":"us-central1-a"},"type":"gce_instance"},"severity":"NOTICE","timestamp":"2020-08-05T16:56:40.428Z"}
+{"insertId":"94170ac4-6e82-4345-98ad-3c780222d19d","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":""},"logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"94170ac4-6e82-4345-98ad-3c780222d19d","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.core.v1.nodes.list","resource":"core/v1/nodes"}],"methodName":"io.k8s.core.v1.nodes.list","requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"GoogleCloudConsole"},"resourceName":"core/v1/nodes","serviceName":"k8s.io","status":{}},"receiveTimestamp":"2021-04-23T14:47:31.94822935Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2021-04-23T14:47:07.535383Z"}
+{"insertId":"b10a904a-faa4-4e0d-9ec3-7bc6a180196a","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"","k8s.io/deprecated":"true","k8s.io/removed-release":"1.22"},"logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"b10a904a-faa4-4e0d-9ec3-7bc6a180196a","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.extensions.v1beta1.ingresses.list","resource":"extensions/v1beta1/namespaces/cos-auditd/ingresses"}],"methodName":"io.k8s.extensions.v1beta1.ingresses.list","requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"GoogleCloudConsole"},"resourceName":"extensions/v1beta1/namespaces/cos-auditd/ingresses","serviceName":"k8s.io","status":{}},"receiveTimestamp":"2021-04-23T14:16:36.37362467Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2021-04-23T14:16:07.574776Z"}
+{"insertId":"e973134d-b4d5-4e2f-92b8-82bba13fdb92","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:public-info-viewer\" of ClusterRole \"system:public-info-viewer\" to Group \"system:unauthenticated\""},"logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"e973134d-b4d5-4e2f-92b8-82bba13fdb92","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:anonymous"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.get","resource":"readyz"}],"methodName":"io.k8s.get","requestMetadata":{"callerIp":"127.0.0.1","callerSuppliedUserAgent":"kube-probe/1.19+"},"resourceName":"readyz","serviceName":"k8s.io","status":{}},"receiveTimestamp":"2021-04-29T08:19:21.606980385Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2021-04-29T08:19:20.80581Z"}
+{"insertId":"03adfb9f-71a3-4f41-9701-29b5542f4d22","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""},"logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"03adfb9f-71a3-4f41-9701-29b5542f4d22","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:serviceaccount:kube-system:generic-garbage-collector"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.get","resource":"api/v1"}],"methodName":"io.k8s.get","requestMetadata":{"callerIp":"::1","callerSuppliedUserAgent":"kube-controller-manager/v1.19.8 (linux/amd64) kubernetes/4f6f69f/system:serviceaccount:kube-system:generic-garbage-collector"},"resourceName":"api/v1","serviceName":"k8s.io","status":{}},"receiveTimestamp":"2021-04-29T08:23:19.71757101Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2021-04-29T08:23:18.899153Z"}