Filebeat azure module pipeline fixes and changes

CHANGELOG.next.asciidoc

@@ -381,6 +381,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `checkpoint.action_reason` when its a string, not a Long. {issue}25575[25575] {pull}25609[25609]
 - Fix `fortinet.firewall.addr` when its a string, not an IP address. {issue}25585[25585] {pull}25608[25608]
 - Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
+- Add improvements to the azure activitylogs and platformlogs ingest pipelines. {pull}26148[26148]
 - Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
 
 *Heartbeat*

filebeat/docs/fields.asciidoc

@@ -2675,30 +2675,13 @@ type: keyword
 
 --
 
-[float]
-=== properties
-
-Properties
-
-
-
-*`azure.activitylogs.properties.service_request_id`*::
-+
---
-Service Request Id
-
-
-type: keyword
-
---
-
-*`azure.activitylogs.properties.status_code`*::
+*`azure.activitylogs.properties`*::
 +
 --
-Status code
+Properties
 
 
-type: keyword
+type: flattened
 
 --
 
@@ -3198,13 +3181,13 @@ type: keyword
 
 --
 
-*`azure.platformlogs.properties.*`*::
+*`azure.platformlogs.properties`*::
 +
 --
-Properties
+Event inner properties
 
 
-type: object
+type: flattened
 
 --
 

x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml

@@ -104,16 +104,6 @@
       description: >
         Event Category
     - name: properties
-      type: group
+      type: flattened
       description: >
         Properties
-      fields:
-        - name: service_request_id
-          type: keyword
-          description: >
-            Service Request Id
-        - name: status_code
-          type: keyword
-          description: >
-            Status code
-

x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml

@@ -12,6 +12,11 @@ processors:
     params:
       empty_field_name: '"":"",'
     ignore_failure: true
+- gsub:
+    field: message
+    pattern: "\\e"
+    replacement: ""
+    ignore_missing: true
 - json:
     field: message
     target_field: azure.activitylogs
@@ -31,9 +36,17 @@ processors:
     field: azure.activitylogs.resourceId
     target_field: azure.resource_id
     ignore_missing: true
-- rename:
+- grok:
+    field: azure.activitylogs.callerIpAddress
+    patterns:
+      - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int}
+      - "%{IPORHOST:source.ip}:%{INT:source.port:int}"
+      - "%{IPORHOST:source.ip}"
+    ignore_missing: true
+    ignore_failure: true
+- remove:
     field: azure.activitylogs.callerIpAddress
-    target_field: source.ip
+    if: 'ctx.source?.ip != null'
     ignore_missing: true
 - set:
     field: client.ip
@@ -63,6 +76,14 @@ processors:
     field: azure.activitylogs.location
     target_field: geo.name
     ignore_missing: true
+- json:
+    field: azure.activitylogs.identity
+    if: "ctx.azure?.activitylogs?.identity instanceof String"
+    ignore_failure: true
+- json:
+    field: azure.activitylogs.properties
+    if: "ctx.azure?.activitylogs?.properties instanceof String"
+    ignore_failure: true
 - script:
     lang: painless
     source: >-
@@ -76,6 +97,10 @@ processors:
         ctx.azure.activitylogs.event_category = 'Administrative';
       }
     ignore_failure: true
+- remove:
+    field: azure.activitylogs.properties.eventCategory
+    if: 'ctx.azure.activitylogs.event_category != null'
+    ignore_missing: true
 - rename:
     field: azure.activitylogs.resultType
     target_field: azure.activitylogs.result_type

x-pack/filebeat/module/azure/activitylogs/test/activitylogs_edgecases.log

@@ -0,0 +1 @@
+{"category":"ResourceHealth","correlationId":"1c867fe2-050c-4a74-bb1c-a83b15246fdd","level":"Information","operationName":"Microsoft.Resourcehealth/healthevent/Updated/action","properties":{"eventCategory":"ResourceHealth","eventProperties":{"cause":"PlatformInitiated"}},"resourceId":"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration","resultType":"Updated","time":"2021-05-25T22:04:07.22Z"}

x-pack/filebeat/module/azure/activitylogs/test/activitylogs_edgecases.log-expected.json

@@ -0,0 +1,28 @@
+[
+    {
+        "@timestamp": "2021-05-25T22:04:07.220Z",
+        "azure.activitylogs.category": "ResourceHealth",
+        "azure.activitylogs.event_category": "ResourceHealth",
+        "azure.activitylogs.operation_name": "Microsoft.Resourcehealth/healthevent/Updated/action",
+        "azure.activitylogs.properties.eventProperties.cause": "PlatformInitiated",
+        "azure.activitylogs.result_type": "Updated",
+        "azure.correlation_id": "1c867fe2-050c-4a74-bb1c-a83b15246fdd",
+        "azure.resource.id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration",
+        "azure.resource.provider": "Microsoft.domainRegistration",
+        "azure.subscription_id": "00000000-0000-0000-0000-000000000000",
+        "cloud.provider": "azure",
+        "event.action": "Microsoft.Resourcehealth/healthevent/Updated/action",
+        "event.dataset": "azure.activitylogs",
+        "event.kind": "event",
+        "event.module": "azure",
+        "event.original": "{\"category\":\"ResourceHealth\",\"correlationId\":\"1c867fe2-050c-4a74-bb1c-a83b15246fdd\",\"level\":\"Information\",\"operationName\":\"Microsoft.Resourcehealth/healthevent/Updated/action\",\"properties\":{\"eventCategory\":\"ResourceHealth\",\"eventProperties\":{\"cause\":\"PlatformInitiated\"}},\"resourceId\":\"/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration\",\"resultType\":\"Updated\",\"time\":\"2021-05-25T22:04:07.22Z\"}",
+        "fileset.name": "activitylogs",
+        "input.type": "log",
+        "log.level": "Information",
+        "log.offset": 0,
+        "service.type": "azure",
+        "tags": [
+            "forwarded"
+        ]
+    }
+]

x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml

@@ -83,6 +83,27 @@ processors:
     field: azure.auditlogs.properties.additionalDetails
     target_field: azure.auditlogs.properties.additional_details
     ignore_missing: true
+- grok:
+    field: azure.auditlogs.callerIpAddress
+    patterns:
+      - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int}
+      - "%{IPORHOST:source.ip}:%{INT:source.port:int}"
+      - "%{IPORHOST:source.ip}"
+    ignore_missing: true
+    ignore_failure: true
+- remove:
+    field: azure.auditlogs.callerIpAddress
+    if: 'ctx.source?.ip != null'
+    ignore_missing: true
+- set:
+    field: client.ip
+    value: '{{source.ip}}'
+    ignore_empty_value: true
+- append:
+    field: related.ip
+    value: '{{source.ip}}'
+    allow_duplicates: false
+    if: 'ctx.source?.ip != null'
 - rename:
     field: azure.auditlogs.properties.resultReason
     target_field: azure.auditlogs.properties.result_reason
@@ -134,6 +155,26 @@ processors:
     field: azure.auditlogs.properties.initiatedBy
     target_field: azure.auditlogs.properties.initiated_by
     ignore_missing: true
+- geoip:
+    field: source.ip
+    target_field: source.geo
+    ignore_missing: true
+- geoip:
+    database_file: GeoLite2-ASN.mmdb
+    field: source.ip
+    target_field: source.as
+    properties:
+      - asn
+      - organization_name
+    ignore_missing: true
+- rename:
+    field: source.as.asn
+    target_field: source.as.number
+    ignore_missing: true
+- rename:
+    field: source.as.organization_name
+    target_field: source.as.organization.name
+    ignore_missing: true
 - set:
     field: event.kind
     value: event

x-pack/filebeat/module/azure/azure-shared-pipeline.yml

@@ -7,6 +7,7 @@ processors:
     field: azure.resource_id
     patterns:
     - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}
+    - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/namespaces/%{NAMESPACE:azure.resource.namespace}/authorizationRules/%{RULE:azure.resource.authorization_rule}
     pattern_definitions:
       SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
       GROUPID: .+
@@ -16,21 +17,66 @@ processors:
     ignore_failure: true
 - grok:
     field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
     patterns:
     - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+    - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
     pattern_definitions:
       SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
       GROUPID: .+
-      PROVIDERNAME: ([A-Z])\w+.([A-Z])\w+/([A-Z])\w+.
+      PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+/([A-Za-z])\w+.
       NAME: ((?!AUTHORIZATIONRULES).)*$
     ignore_failure: true
 - grok:
     field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
+    patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+      - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+    pattern_definitions:
+      SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+      GROUPID: .+
+      PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+
+      NAME: .+
+    ignore_failure: true
+- grok:
+    field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
     patterns:
     - /providers/%{PROVIDER:azure.resource.provider}
+    - /PROVIDERS/%{PROVIDER:azure.resource.provider}
     pattern_definitions:
       PROVIDER: .+
     ignore_failure: true
+- grok:
+    field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
+    patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}
+      - /subscriptions/%{SUBID:azure.subscription_id}/providers/%{PROVIDERNAME:azure.resource.provider}
+    pattern_definitions:
+      SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+      PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+
+    ignore_failure: true
+- grok:
+    field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
+    patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}
+      - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}
+    pattern_definitions:
+      SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+      GROUPID: .+
+    ignore_failure: true
+- grok:
+    field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
+    patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}
+      - /subscriptions/%{SUBID:azure.subscription_id}
+    pattern_definitions:
+      SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+    ignore_failure: true
 - rename:
     field: azure.resource_id
     target_field: azure.resource.id

x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml

@@ -57,10 +57,7 @@
       type: keyword
       description: >
         ActivityId
-    - name: properties.*
-      type: object
-      object_type: keyword
-      object_type_mapping_type: "*"
+    - name: properties
+      type: flattened
       description: >
-        Properties
-
+        Event inner properties

x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml

@@ -51,17 +51,30 @@ processors:
     field:
       - azure.platformlogs.EventProperties
     ignore_missing: true
-- rename:
-    field: azure.platformlogs.EventName
-    target_field: event.action
-    ignore_missing: true
+- json:
+    field: azure.platformlogs.properties.log
+    target_field: azure.platformlogs.properties.log
+    ignore_failure: true
 - rename:
     field: azure.platformlogs.properties.log
+    if: "ctx.azure?.platformlogs?.properties?.log != null && ctx.azure?.platformlogs?.properties?.log instanceof String"
     target_field: message
     ignore_missing: true
 - rename:
+    field: azure.platformlogs.EventName
+    target_field: event.action
+    ignore_missing: true
+- grok:
     field: azure.platformlogs.callerIpAddress
-    target_field: source.ip
+    patterns:
+      - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int}
+      - "%{IPORHOST:source.ip}:%{INT:source.port:int}"
+      - "%{IPORHOST:source.ip}"
+    ignore_missing: true
+    ignore_failure: true
+- remove:
+    field: azure.platformlogs.callerIpAddress
+    if: 'ctx.source?.ip != null'
     ignore_missing: true
 - set:
     field: client.ip
@@ -174,10 +187,6 @@ processors:
       }
       def hm = new HashMap(params.get(ctx.azure.platformlogs.category.toLowerCase()));
       hm.forEach((k, v) -> ctx.event[k] = v);
-- geoip:
-    field: source.ip
-    target_field: source.geo
-    ignore_missing: true
 - geoip:
     database_file: GeoLite2-ASN.mmdb
     field: source.ip