Filebeat azure module pipeline fixes and changes

CHANGELOG.next.asciidoc

@@ -381,6 +381,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `checkpoint.action_reason` when its a string, not a Long. {issue}25575[25575] {pull}25609[25609]
 - Fix `fortinet.firewall.addr` when its a string, not an IP address. {issue}25585[25585] {pull}25608[25608]
 - Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
+- Add improvements to the azure activitylogs and platformlogs ingest pipelines. {pull}26148[26148]
 - Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
 
 *Heartbeat*

filebeat/docs/fields.asciidoc

@@ -2698,6 +2698,23 @@ type: keyword
 Status code
 
 
+type: keyword
+
+--
+
+[float]
+=== eventProperties
+
+Event Properties
+
+
+
+*`azure.activitylogs.properties.eventProperties.cause`*::
++
+--
+Cause
+
+
 type: keyword
 
 --
@@ -3198,13 +3215,57 @@ type: keyword
 
 --
 
-*`azure.platformlogs.properties.*`*::
+[float]
+=== properties
+
+Event inner properties
+
+
+
+*`azure.platformlogs.properties.Namespace`*::
 +
 --
-Properties
+Namespace
 
 
-type: object
+type: keyword
+
+--
+
+*`azure.platformlogs.properties.SubscriptionId`*::
++
+--
+Subscription Id
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.properties.pod`*::
++
+--
+Pod Id
+
+
+type: keyword
+
+--
+
+[float]
+=== log
+
+Log properties
+
+
+
+*`azure.platformlogs.properties.log.kind`*::
++
+--
+Type of message
+
+
+type: keyword
 
 --
 

x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml

@@ -116,4 +116,12 @@
           type: keyword
           description: >
             Status code
-
+        - name: eventProperties
+          type: group
+          description: >
+            Event Properties
+          fields:
+            - name: cause
+              type: keyword
+              description: >
+                Cause

x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml

@@ -12,6 +12,11 @@ processors:
     params:
       empty_field_name: '"":"",'
     ignore_failure: true
+- gsub:
+    field: message
+    pattern: "\\e"
+    replacement: ""
+    ignore_missing: true
 - json:
     field: message
     target_field: azure.activitylogs
@@ -31,9 +36,17 @@ processors:
     field: azure.activitylogs.resourceId
     target_field: azure.resource_id
     ignore_missing: true
-- rename:
+- grok:
+    field: azure.activitylogs.callerIpAddress
+    patterns:
+      - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int}
+      - "%{IPORHOST:source.ip}:%{INT:source.port:int}"
+      - "%{IPORHOST:source.ip}"
+    ignore_missing: true
+    ignore_failure: true
+- remove:
     field: azure.activitylogs.callerIpAddress
-    target_field: source.ip
+    if: 'ctx.source?.ip != null'
     ignore_missing: true
 - set:
     field: client.ip
@@ -63,6 +76,14 @@ processors:
     field: azure.activitylogs.location
     target_field: geo.name
     ignore_missing: true
+- json:
+    field: azure.activitylogs.identity
+    if: "ctx.azure?.activitylogs?.identity instanceof String"
+    ignore_failure: true
+- json:
+    field: azure.activitylogs.properties
+    if: "ctx.azure?.activitylogs?.properties instanceof String"
+    ignore_failure: true
 - script:
     lang: painless
     source: >-
@@ -76,6 +97,10 @@ processors:
         ctx.azure.activitylogs.event_category = 'Administrative';
       }
     ignore_failure: true
+- remove:
+    field: azure.activitylogs.properties.eventCategory
+    if: 'ctx.azure.activitylogs.event_category != null'
+    ignore_missing: true
 - rename:
     field: azure.activitylogs.resultType
     target_field: azure.activitylogs.result_type

x-pack/filebeat/module/azure/activitylogs/test/activitylogs_edgecases.log

@@ -0,0 +1 @@
+{"category":"ResourceHealth","correlationId":"1c867fe2-050c-4a74-bb1c-a83b15246fdd","level":"Information","operationName":"Microsoft.Resourcehealth/healthevent/Updated/action","properties":{"eventCategory":"ResourceHealth","eventProperties":{"cause":"PlatformInitiated"}},"resourceId":"/SUBSCRIPTIONS/00000000-0000-0000-0000-000000000000/PROVIDERS/MICROSOFT.DOMAINREGISTRATION","resultType":"Updated","time":"2021-05-25T22:04:07.22Z"}

x-pack/filebeat/module/azure/activitylogs/test/activitylogs_edgecases.log-expected.json

@@ -0,0 +1,28 @@
+[
+{
+    "log.offset":0,
+    "log.level":"Information",
+    "fileset.name":"activitylogs",
+    "tags":[
+        "forwarded"
+    ],
+    "cloud.provider":"azure",
+    "input.type":"log",
+    "@timestamp":"2021-05-25T22:04:07.220Z",
+    "service.type":"azure",
+    "event.original":"{\"category\":\"ResourceHealth\",\"correlationId\":\"1c867fe2-050c-4a74-bb1c-a83b15246fdd\",\"level\":\"Information\",\"operationName\":\"Microsoft.Resourcehealth/healthevent/Updated/action\",\"properties\":{\"eventCategory\":\"ResourceHealth\",\"eventProperties\":{\"cause\":\"PlatformInitiated\"}},\"resourceId\":\"/SUBSCRIPTIONS/00000000-0000-0000-0000-000000000000/PROVIDERS/MICROSOFT.DOMAINREGISTRATION\",\"resultType\":\"Updated\",\"time\":\"2021-05-25T22:04:07.22Z\"}",
+    "event.kind":"event",
+    "event.module":"azure",
+    "event.action":"Microsoft.Resourcehealth/healthevent/Updated/action",
+    "event.dataset":"azure.activitylogs",
+    "azure.subscription_id":"00000000-0000-0000-0000-000000000000",
+    "azure.resource.provider":"MICROSOFT.DOMAINREGISTRATION",
+    "azure.resource.id":"/SUBSCRIPTIONS/00000000-0000-0000-0000-000000000000/PROVIDERS/MICROSOFT.DOMAINREGISTRATION",
+    "azure.correlation_id":"1c867fe2-050c-4a74-bb1c-a83b15246fdd",
+    "azure.activitylogs.operation_name":"Microsoft.Resourcehealth/healthevent/Updated/action",
+    "azure.activitylogs.result_type":"Updated",
+    "azure.activitylogs.category":"ResourceHealth",
+    "azure.activitylogs.event_category":"ResourceHealth",
+    "azure.activitylogs.properties.eventProperties.cause":"PlatformInitiated"
+}
+    ]

x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml

@@ -83,6 +83,27 @@ processors:
     field: azure.auditlogs.properties.additionalDetails
     target_field: azure.auditlogs.properties.additional_details
     ignore_missing: true
+- grok:
+    field: azure.auditlogs.callerIpAddress
+    patterns:
+      - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int}
+      - "%{IPORHOST:source.ip}:%{INT:source.port:int}"
+      - "%{IPORHOST:source.ip}"
+    ignore_missing: true
+    ignore_failure: true
+- remove:
+    field: azure.auditlogs.callerIpAddress
+    if: 'ctx.source?.ip != null'
+    ignore_missing: true
+- set:
+    field: client.ip
+    value: '{{source.ip}}'
+    ignore_empty_value: true
+- append:
+    field: related.ip
+    value: '{{source.ip}}'
+    allow_duplicates: false
+    if: 'ctx.source?.ip != null'
 - rename:
     field: azure.auditlogs.properties.resultReason
     target_field: azure.auditlogs.properties.result_reason
@@ -134,6 +155,26 @@ processors:
     field: azure.auditlogs.properties.initiatedBy
     target_field: azure.auditlogs.properties.initiated_by
     ignore_missing: true
+- geoip:
+    field: source.ip
+    target_field: source.geo
+    ignore_missing: true
+- geoip:
+    database_file: GeoLite2-ASN.mmdb
+    field: source.ip
+    target_field: source.as
+    properties:
+      - asn
+      - organization_name
+    ignore_missing: true
+- rename:
+    field: source.as.asn
+    target_field: source.as.number
+    ignore_missing: true
+- rename:
+    field: source.as.organization_name
+    target_field: source.as.organization.name
+    ignore_missing: true
 - set:
     field: event.kind
     value: event

x-pack/filebeat/module/azure/azure-shared-pipeline.yml

@@ -7,6 +7,7 @@ processors:
     field: azure.resource_id
     patterns:
     - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}
+    - /subscription/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/namespaces/%{NAMESPACE:azure.resource.namespace}/authorizationRules/%{RULE:azure.resource.authorization_rule}
     pattern_definitions:
       SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
       GROUPID: .+
@@ -16,21 +17,66 @@ processors:
     ignore_failure: true
 - grok:
     field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
     patterns:
     - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+    - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
     pattern_definitions:
       SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
       GROUPID: .+
-      PROVIDERNAME: ([A-Z])\w+.([A-Z])\w+/([A-Z])\w+.
+      PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+/([A-Za-z])\w+.
       NAME: ((?!AUTHORIZATIONRULES).)*$
     ignore_failure: true
 - grok:
     field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
+    patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+      - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+    pattern_definitions:
+      SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+      GROUPID: .+
+      PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+
+      NAME: .+
+    ignore_failure: true
+- grok:
+    field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
     patterns:
     - /providers/%{PROVIDER:azure.resource.provider}
+    - /PROVIDERS/%{PROVIDER:azure.resource.provider}
     pattern_definitions:
       PROVIDER: .+
     ignore_failure: true
+- grok:
+    field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
+    patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}
+      - /subscriptions/%{SUBID:azure.subscription_id}/providers/%{PROVIDERNAME:azure.resource.provider}
+    pattern_definitions:
+      SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+      PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+
+    ignore_failure: true
+- grok:
+    field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
+    patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}
+      - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}
+    pattern_definitions:
+      SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+      GROUPID: .+
+    ignore_failure: true
+- grok:
+    field: azure.resource_id
+    if: 'ctx.azure?.subscription_id == null'
+    patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}
+      - /subscriptions/%{SUBID:azure.subscription_id}
+    pattern_definitions:
+      SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+    ignore_failure: true
 - rename:
     field: azure.resource_id
     target_field: azure.resource.id

x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml

@@ -57,10 +57,29 @@
       type: keyword
       description: >
         ActivityId
-    - name: properties.*
-      type: object
-      object_type: keyword
-      object_type_mapping_type: "*"
+    - name: properties
+      type: group
       description: >
-        Properties
-
+        Event inner properties
+      fields:
+        - name: Namespace
+          type: keyword
+          description: >
+            Namespace
+        - name: SubscriptionId
+          type: keyword
+          description: >
+            Subscription Id
+        - name: pod
+          type: keyword
+          description: >
+            Pod Id
+        - name: log
+          type: group
+          description: >
+            Log properties
+          fields:
+            - name: kind
+              type: keyword
+              description: >
+                Type of message