Cherry-pick #26148 to 7.x: Filebeat azure module pipeline fixes and c…

…hanges (#26233) * Filebeat azure module pipeline fixes and changes (#26148) * fix for filbeat pipeline * changelog * fixes * retest * generate files * retest * fix tests * fix tests * update fields * update result * fix name * Format expected json files * sub * update fields * fix test Co-authored-by: Jaime Soriano Pastor <jaime.soriano@elastic.co> (cherry picked from commit d971550) * generate fields
elastic · Jun 10, 2021 · 5f21f7c · 5f21f7c
1 parent 6ed7ae7
commit 5f21f7c
Show file tree

Hide file tree

Showing 16 changed files with 189 additions and 63 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -305,6 +305,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove space from field `sophos.xg.trans_src_ ip`. {issue}25154[25154] {pull}25250[25250]
 - Fix `checkpoint.action_reason` when its a string, not a Long. {issue}25575[25575] {pull}25609[25609]
 - Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
+- Add improvements to the azure activitylogs and platformlogs ingest pipelines. {pull}26148[26148]
 - Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
 - o365: Avoid mapping exception for `Parameters` and `ExtendedProperties` fields of string type. {pull}26164[26164]
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -2675,30 +2675,13 @@ type: keyword
 
 --
 
-[float]
-=== properties
-
-Properties
-
-
-
-*`azure.activitylogs.properties.service_request_id`*::
-+
---
-Service Request Id
-
-
-type: keyword
-
---
-
-*`azure.activitylogs.properties.status_code`*::
+*`azure.activitylogs.properties`*::
 +
 --
-Status code
+Properties
 
 
-type: keyword
+type: flattened
 
 --
 
@@ -3198,13 +3181,13 @@ type: keyword
 
 --
 
-*`azure.platformlogs.properties.*`*::
+*`azure.platformlogs.properties`*::
 +
 --
-Properties
+Event inner properties
 
 
-type: object
+type: flattened
 
 --
 

diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml
@@ -104,16 +104,6 @@
       description: >
         Event Category
     - name: properties
-      type: group
+      type: flattened
       description: >
         Properties
-      fields:
-        - name: service_request_id
-          type: keyword
-          description: >
-            Service Request Id
-        - name: status_code
-          type: keyword
-          description: >
-            Status code
-
diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml
@@ -12,6 +12,11 @@ processors:
     params:
       empty_field_name: '"":"",'
     ignore_failure: true
+- gsub:
+    field: message
+    pattern: "\\e"
+    replacement: ""
+    ignore_missing: true
 - json:
     field: message
     target_field: azure.activitylogs
@@ -31,9 +36,17 @@ processors:
     field: azure.activitylogs.resourceId
     target_field: azure.resource_id
     ignore_missing: true
-- rename:
+- grok:
+    field: azure.activitylogs.callerIpAddress
+    patterns:
+      - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int}
+      - "%{IPORHOST:source.ip}:%{INT:source.port:int}"
+      - "%{IPORHOST:source.ip}"
+    ignore_missing: true
+    ignore_failure: true
+- remove:
     field: azure.activitylogs.callerIpAddress
-    target_field: source.ip
+    if: 'ctx.source?.ip != null'
     ignore_missing: true
 - set:
     field: client.ip
@@ -63,6 +76,14 @@ processors:
     field: azure.activitylogs.location
     target_field: geo.name
     ignore_missing: true
+- json:
+    field: azure.activitylogs.identity
+    if: "ctx.azure?.activitylogs?.identity instanceof String"
+    ignore_failure: true
+- json:
+    field: azure.activitylogs.properties
+    if: "ctx.azure?.activitylogs?.properties instanceof String"
+    ignore_failure: true
 - script:
     lang: painless
     source: >-
@@ -76,6 +97,10 @@ processors:
         ctx.azure.activitylogs.event_category = 'Administrative';
       }
     ignore_failure: true
+- remove:
+    field: azure.activitylogs.properties.eventCategory
+    if: 'ctx.azure.activitylogs.event_category != null'
+    ignore_missing: true
 - rename:
     field: azure.activitylogs.resultType
     target_field: azure.activitylogs.result_type

diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activitylogs_edgecases.log b/x-pack/filebeat/module/azure/activitylogs/test/activitylogs_edgecases.log
@@ -0,0 +1 @@
+{"category":"ResourceHealth","correlationId":"1c867fe2-050c-4a74-bb1c-a83b15246fdd","level":"Information","operationName":"Microsoft.Resourcehealth/healthevent/Updated/action","properties":{"eventCategory":"ResourceHealth","eventProperties":{"cause":"PlatformInitiated"}},"resourceId":"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration","resultType":"Updated","time":"2021-05-25T22:04:07.22Z"}
diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activitylogs_edgecases.log-expected.json b/x-pack/filebeat/module/azure/activitylogs/test/activitylogs_edgecases.log-expected.json
@@ -0,0 +1,28 @@
+[
+    {
+        "@timestamp": "2021-05-25T22:04:07.220Z",
+        "azure.activitylogs.category": "ResourceHealth",
+        "azure.activitylogs.event_category": "ResourceHealth",
+        "azure.activitylogs.operation_name": "Microsoft.Resourcehealth/healthevent/Updated/action",
+        "azure.activitylogs.properties.eventProperties.cause": "PlatformInitiated",
+        "azure.activitylogs.result_type": "Updated",
+        "azure.correlation_id": "1c867fe2-050c-4a74-bb1c-a83b15246fdd",
+        "azure.resource.id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration",
+        "azure.resource.provider": "Microsoft.domainRegistration",
+        "azure.subscription_id": "00000000-0000-0000-0000-000000000000",
+        "cloud.provider": "azure",
+        "event.action": "Microsoft.Resourcehealth/healthevent/Updated/action",
+        "event.dataset": "azure.activitylogs",
+        "event.kind": "event",
+        "event.module": "azure",
+        "event.original": "{\"category\":\"ResourceHealth\",\"correlationId\":\"1c867fe2-050c-4a74-bb1c-a83b15246fdd\",\"level\":\"Information\",\"operationName\":\"Microsoft.Resourcehealth/healthevent/Updated/action\",\"properties\":{\"eventCategory\":\"ResourceHealth\",\"eventProperties\":{\"cause\":\"PlatformInitiated\"}},\"resourceId\":\"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration\",\"resultType\":\"Updated\",\"time\":\"2021-05-25T22:04:07.22Z\"}",
+        "fileset.name": "activitylogs",
+        "input.type": "log",
+        "log.level": "Information",
+        "log.offset": 0,
+        "service.type": "azure",
+        "tags": [
+            "forwarded"
+        ]
+    }
+]
diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml
@@ -83,6 +83,27 @@ processors:
     field: azure.auditlogs.properties.additionalDetails
     target_field: azure.auditlogs.properties.additional_details
     ignore_missing: true
+- grok:
+    field: azure.auditlogs.callerIpAddress
+    patterns:
+      - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int}
+      - "%{IPORHOST:source.ip}:%{INT:source.port:int}"
+      - "%{IPORHOST:source.ip}"
+    ignore_missing: true
+    ignore_failure: true
+- remove:
+    field: azure.auditlogs.callerIpAddress
+    if: 'ctx.source?.ip != null'
+    ignore_missing: true
+- set:
+    field: client.ip
+    value: '{{source.ip}}'
+    ignore_empty_value: true
+- append:
+    field: related.ip
+    value: '{{source.ip}}'
+    allow_duplicates: false
+    if: 'ctx.source?.ip != null'
 - rename:
     field: azure.auditlogs.properties.resultReason
     target_field: azure.auditlogs.properties.result_reason
@@ -134,6 +155,26 @@ processors:
     field: azure.auditlogs.properties.initiatedBy
     target_field: azure.auditlogs.properties.initiated_by
     ignore_missing: true
+- geoip:
+    field: source.ip
+    target_field: source.geo
+    ignore_missing: true
+- geoip:
+    database_file: GeoLite2-ASN.mmdb
+    field: source.ip
+    target_field: source.as
+    properties:
+      - asn
+      - organization_name
+    ignore_missing: true
+- rename:
+    field: source.as.asn
+    target_field: source.as.number
+    ignore_missing: true
+- rename:
+    field: source.as.organization_name
+    target_field: source.as.organization.name
+    ignore_missing: true
 - set:
     field: event.kind
     value: event

diff --git a/x-pack/filebeat/module/azure/azure-shared-pipeline.yml b/x-pack/filebeat/module/azure/azure-shared-pipeline.yml
@@ -7,6 +7,7 @@ processors:
     field: azure.resource_id
     patterns:
     - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}
+    - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/namespaces/%{NAMESPACE:azure.resource.namespace}/authorizationRules/%{RULE:azure.resource.authorization_rule}
     pattern_definitions:
       SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
       GROUPID: .+
@@ -16,21 +17,66 @@ processors:
     ignore_failure: true
 - grok:
     field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
     patterns:
     - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+    - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
     pattern_definitions:
       SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
       GROUPID: .+
-      PROVIDERNAME: ([A-Z])\w+.([A-Z])\w+/([A-Z])\w+.
+      PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+/([A-Za-z])\w+.
       NAME: ((?!AUTHORIZATIONRULES).)*$
     ignore_failure: true
 - grok:
     field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
+    patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+      - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+    pattern_definitions:
+      SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+      GROUPID: .+
+      PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+
+      NAME: .+
+    ignore_failure: true
+- grok:
+    field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
     patterns:
     - /providers/%{PROVIDER:azure.resource.provider}
+    - /PROVIDERS/%{PROVIDER:azure.resource.provider}
     pattern_definitions:
       PROVIDER: .+
     ignore_failure: true
+- grok:
+    field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
+    patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}
+      - /subscriptions/%{SUBID:azure.subscription_id}/providers/%{PROVIDERNAME:azure.resource.provider}
+    pattern_definitions:
+      SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+      PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+
+    ignore_failure: true
+- grok:
+    field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
+    patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}
+      - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}
+    pattern_definitions:
+      SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+      GROUPID: .+
+    ignore_failure: true
+- grok:
+    field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
+    patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}
+      - /subscriptions/%{SUBID:azure.subscription_id}
+    pattern_definitions:
+      SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+    ignore_failure: true
 - rename:
     field: azure.resource_id
     target_field: azure.resource.id

diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go
diff --git a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml
@@ -57,10 +57,7 @@
       type: keyword
       description: >
         ActivityId
-    - name: properties.*
-      type: object
-      object_type: keyword
-      object_type_mapping_type: "*"
+    - name: properties
+      type: flattened
       description: >
-        Properties
-
+        Event inner properties
diff --git a/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml
@@ -51,17 +51,30 @@ processors:
     field:
       - azure.platformlogs.EventProperties
     ignore_missing: true
-- rename:
-    field: azure.platformlogs.EventName
-    target_field: event.action
-    ignore_missing: true
+- json:
+    field: azure.platformlogs.properties.log
+    target_field: azure.platformlogs.properties.log
+    ignore_failure: true
 - rename:
     field: azure.platformlogs.properties.log
+    if: "ctx.azure?.platformlogs?.properties?.log != null && ctx.azure?.platformlogs?.properties?.log instanceof String"
     target_field: message
     ignore_missing: true
 - rename:
+    field: azure.platformlogs.EventName
+    target_field: event.action
+    ignore_missing: true
+- grok:
     field: azure.platformlogs.callerIpAddress
-    target_field: source.ip
+    patterns:
+      - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int}
+      - "%{IPORHOST:source.ip}:%{INT:source.port:int}"
+      - "%{IPORHOST:source.ip}"
+    ignore_missing: true
+    ignore_failure: true
+- remove:
+    field: azure.platformlogs.callerIpAddress
+    if: 'ctx.source?.ip != null'
     ignore_missing: true
 - set:
     field: client.ip
@@ -174,10 +187,6 @@ processors:
       }
       def hm = new HashMap(params.get(ctx.azure.platformlogs.category.toLowerCase()));
       hm.forEach((k, v) -> ctx.event[k] = v);
-- geoip:
-    field: source.ip
-    target_field: source.geo
-    ignore_missing: true
 - geoip:
     database_file: GeoLite2-ASN.mmdb
     field: source.ip

diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log
@@ -1 +1 @@
-{"ActivityId":"30ed877c-a36b-491a-bd4d-ddd847fe55b8","Caller":"Portal","Environment":"PROD","EventName":"Retreive ConsumerGroup","EventProperties":"{\"SubscriptionId\":\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\",\"Namespace\":\"obstesteventhubs\",\"Via\":\"sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04\u0026$skip=0\u0026$top=100\",\"TrackingId\":\"30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2\"}","EventTimeString":"11/3/2020 9:06:42 AM +00:00","Region":"West Europe","ScaleUnit":"PROD-AM3-AZ501","Status":"Succeeded","category":"OperationalLogs","resourceId":"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS"}
+{"ActivityId":"30ed877c-a36b-491a-bd4d-ddd847fe55b8","Caller":"Portal","Environment":"PROD","EventName":"Retreive ConsumerGroup","EventProperties":"{\"SubscriptionId\":\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\",\"Namespace\":\"obstesteventhubs\"}","EventTimeString":"11/3/2020 9:06:42 AM +00:00","Region":"West Europe","ScaleUnit":"PROD-AM3-AZ501","Status":"Succeeded","category":"OperationalLogs","resourceId":"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS"}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"category":"ResourceHealth","correlationId":"1c867fe2-050c-4a74-bb1c-a83b15246fdd","level":"Information","operationName":"Microsoft.Resourcehealth/healthevent/Updated/action","properties":{"eventCategory":"ResourceHealth","eventProperties":{"cause":"PlatformInitiated"}},"resourceId":"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration","resultType":"Updated","time":"2021-05-25T22:04:07.22Z"}
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		{"ActivityId":"30ed877c-a36b-491a-bd4d-ddd847fe55b8","Caller":"Portal","Environment":"PROD","EventName":"Retreive ConsumerGroup","EventProperties":"{\"SubscriptionId\":\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\",\"Namespace\":\"obstesteventhubs\",\"Via\":\"sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04\u0026$skip=0\u0026$top=100\",\"TrackingId\":\"30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2\"}","EventTimeString":"11/3/2020 9:06:42 AM +00:00","Region":"West Europe","ScaleUnit":"PROD-AM3-AZ501","Status":"Succeeded","category":"OperationalLogs","resourceId":"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS"}
		{"ActivityId":"30ed877c-a36b-491a-bd4d-ddd847fe55b8","Caller":"Portal","Environment":"PROD","EventName":"Retreive ConsumerGroup","EventProperties":"{\"SubscriptionId\":\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\",\"Namespace\":\"obstesteventhubs\"}","EventTimeString":"11/3/2020 9:06:42 AM +00:00","Region":"West Europe","ScaleUnit":"PROD-AM3-AZ501","Status":"Succeeded","category":"OperationalLogs","resourceId":"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS"}