[MS365] [Subscriptions] Add Subscriptions data stream (elastic#12490)

harnish-elastic · Feb 5, 2025 · 7c7e061 · 7c7e061
1 parent 67cbd45
commit 7c7e061
Show file tree

Hide file tree

Showing 11 changed files with 648 additions and 0 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -480,3 +480,4 @@
 /packages/o365_metrics/data_stream/yammer_device_usage @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/service_health @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/viva_engage_device_usage_user_counts @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/subscriptions @elastic/obs-infraobs-integrations
diff --git a/packages/o365_metrics/data_stream/subscriptions/_dev/test/pipeline/test-common-config.yml b/packages/o365_metrics/data_stream/subscriptions/_dev/test/pipeline/test-common-config.yml
@@ -0,0 +1,5 @@
+dynamic_fields:
+  "event.ingested": ".*"
+fields:
+  tags:
+    - preserve_original_event
diff --git a/packages/o365_metrics/data_stream/subscriptions/_dev/test/pipeline/test-subscriptions.json b/packages/o365_metrics/data_stream/subscriptions/_dev/test/pipeline/test-subscriptions.json
@@ -0,0 +1,8 @@
+{
+    "events": [
+        {
+            "message": "{\"accountName\":\"Account1\",\"accountId\":\"f97aeefc-af85-414d-8ae4-b457f90efc40\",\"appliesTo\":\"User\",\"capabilityStatus\":\"Enabled\",\"consumedUnits\":14,\"id\":\"48a80680-7326-48cd-9935-b556b81d3a4e_c7df2760-2c81-4ef7-b578-5b5392b571df\",\"prepaidUnits\":{\"enabled\":25,\"lockedOut\":5,\"suspended\":13,\"warning\":7},\"servicePlans\":[{\"servicePlanId\":\"8c098270-9dd4-4350-9b30-ba4703f3b36b\",\"servicePlanName\":\"ADALLOM_S_O365\",\"provisioningStatus\":\"Success\",\"appliesTo\":\"Company\"}],\"skuId\":\"c7df2760-2c81-4ef7-b578-5b5392b571df\",\"skuPartNumber\":\"ENTERPRISEPREMIUM\",\"subscriptionIds\":[\"43d26afe-cb98-48b9-acc4-ae3ef2ac6c51\"]}"
+
+        }
+    ]
+}
diff --git a/...etrics/data_stream/subscriptions/_dev/test/pipeline/test-subscriptions.json-expected.json b/...etrics/data_stream/subscriptions/_dev/test/pipeline/test-subscriptions.json-expected.json
@@ -0,0 +1,59 @@
+{
+    "expected": [
+        {
+            "ecs": {
+                "version": "8.16.0"
+            },
+            "event": {
+                "original": "{\"accountName\":\"Account1\",\"accountId\":\"f97aeefc-af85-414d-8ae4-b457f90efc40\",\"appliesTo\":\"User\",\"capabilityStatus\":\"Enabled\",\"consumedUnits\":14,\"id\":\"48a80680-7326-48cd-9935-b556b81d3a4e_c7df2760-2c81-4ef7-b578-5b5392b571df\",\"prepaidUnits\":{\"enabled\":25,\"lockedOut\":5,\"suspended\":13,\"warning\":7},\"servicePlans\":[{\"servicePlanId\":\"8c098270-9dd4-4350-9b30-ba4703f3b36b\",\"servicePlanName\":\"ADALLOM_S_O365\",\"provisioningStatus\":\"Success\",\"appliesTo\":\"Company\"}],\"skuId\":\"c7df2760-2c81-4ef7-b578-5b5392b571df\",\"skuPartNumber\":\"ENTERPRISEPREMIUM\",\"subscriptionIds\":[\"43d26afe-cb98-48b9-acc4-ae3ef2ac6c51\"]}"
+            },
+            "o365": {
+                "metrics": {
+                    "subscriptions": {
+                        "account_id": "f97aeefc-af85-414d-8ae4-b457f90efc40",
+                        "account_name": "Account1",
+                        "applies_to": "User",
+                        "capability_status": "Enabled",
+                        "consumed_units": {
+                            "count": 14
+                        },
+                        "id": "48a80680-7326-48cd-9935-b556b81d3a4e_c7df2760-2c81-4ef7-b578-5b5392b571df",
+                        "prepaid_units": {
+                            "enabled": {
+                                "count": 25
+                            },
+                            "locked_out": {
+                                "count": 5
+                            },
+                            "suspended": {
+                                "count": 13
+                            },
+                            "warning": {
+                                "count": 7
+                            }
+                        },
+                        "service_plans": [
+                            {
+                                "applies_to": "Company",
+                                "provisioning_status": "Success",
+                                "service_plan_id": "8c098270-9dd4-4350-9b30-ba4703f3b36b",
+                                "service_plan_name": "ADALLOM_S_O365"
+                            }
+                        ],
+                        "sku_id": "c7df2760-2c81-4ef7-b578-5b5392b571df",
+                        "sku_part_number": "ENTERPRISEPREMIUM",
+                        "subscription_ids": [
+                            "43d26afe-cb98-48b9-acc4-ae3ef2ac6c51"
+                        ],
+                        "surplus_units": {
+                            "count": 11
+                        }
+                    }
+                }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        }
+    ]
+}
diff --git a/packages/o365_metrics/data_stream/subscriptions/agent/stream/cel.yml.hbs b/packages/o365_metrics/data_stream/subscriptions/agent/stream/cel.yml.hbs
@@ -0,0 +1,82 @@
+config_version: 2
+interval: {{interval}}
+auth.oauth2:
+    client.id: {{client_id}}
+    client.secret: {{client_secret}}
+    provider: azure
+    scopes:
+{{#each token_scopes as |token_scope|}}
+      - {{token_scope}}
+{{/each}}
+    endpoint_params: 
+        grant_type: client_credentials
+{{#if token_url}}
+    token_url: {{token_url}}/{{azure_tenant_id}}/oauth2/v2.0/token
+{{else if azure_tenant_id}}
+    azure.tenant_id: {{azure_tenant_id}}
+{{/if}}
+
+resource.url: {{url}}
+{{#if resource_ssl}}
+resource.ssl: 
+  {{resource_ssl}}
+{{/if}}
+
+{{#if enable_request_tracer}}
+resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson"
+{{/if}}
+
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
+
+state:
+  want_more: false
+  base:
+    tenant_id: "{{azure_tenant_id}}"
+    period: "{{period}}"
+
+redact:
+  fields:
+    - base.tenant_id
+
+program: |
+    state.with(
+      request(
+        "GET",
+        state.url + "/subscribedSkus"
+      ).do_request().as(resp,
+        resp.StatusCode == 200
+        ?
+          bytes(resp.Body).decode_json().as(body, {
+                 "events": body.value.map(e,{"message": e.encode_json()}),                   
+          })
+        :
+          {
+            "events": {
+              "error": {
+                "code": string(resp.StatusCode),
+                "id": string(resp.Status),
+                "message": "GET:"+(
+                  size(resp.Body) != 0 ?
+                    string(resp.Body)
+                  :
+                    string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+                ),
+              },
+            },
+            "want_more": false,
+          }
+      )
+    )