[System] Add dimensions to system package metrics data_streams only, …

…except core data_streams (elastic#6118) * add dimensions to system package, metrics data_streams only Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * revert k8s package Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * revert k8s changes; add extra dimensions for diskio and network Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * filesystem: add mount_point and device_name Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * move core and process datastream to different PR Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * update PR number Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * move agent.id field to ecs.yaml Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * run elastic-package build Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * run elastic-package build Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * elastic-package check Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * add dimensions to process data_stream Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * clean up some duplicated fields Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * revert network data_stream changes Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * rever process data_stream changes Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * adjust the changelog description Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * revert process data_stream changes Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> --------- Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>
agithomas · Jun 5, 2023 · 509a387 · 509a387
1 parent 6401efd
commit 509a387
Show file tree

Hide file tree

Showing 24 changed files with 99 additions and 59 deletions.
diff --git a/packages/system/changelog.yml b/packages/system/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.30.0"
+  changes:
+    - description: Add dimension fields to metrics all data_streams, except core, network and process to support TSDS migration
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6118
 - version: "1.29.0"
   changes:
     - description: support ip or domain in sshd messages

diff --git a/packages/system/data_stream/cpu/fields/agent.yml b/packages/system/data_stream/cpu/fields/agent.yml
@@ -13,12 +13,14 @@
 
         Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.'
       example: 666777888999
+      dimension: true
     - name: availability_zone
       level: extended
       type: keyword
       ignore_above: 1024
       description: Availability zone in which this host is running.
       example: us-east-1c
+      dimension: true
     - name: instance.id
       level: extended
       type: keyword
@@ -50,10 +52,10 @@
       ignore_above: 1024
       description: Region in which this host is running.
       example: us-east-1
+      dimension: true
     - name: project.id
       type: keyword
       description: Name of the project in Google Cloud.
-      dimension: true
     - name: image.id
       type: keyword
       description: Image ID for the cloud instance.
@@ -138,10 +140,10 @@
       level: core
       type: keyword
       ignore_above: 1024
-      dimension: true
       description: 'Name of the host.
 
         It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.'
+      dimension: true
     - name: os.family
       level: extended
       type: keyword
@@ -201,11 +203,3 @@
       description: >
         OS codename, if any.
 
-- name: agent
-  title: Agent
-  type: group
-  fields:
-    - name: id
-      type: keyword
-      ignore_above: 1024
-      dimension: true
diff --git a/packages/system/data_stream/cpu/fields/ecs.yml b/packages/system/data_stream/cpu/fields/ecs.yml
@@ -22,3 +22,6 @@
   name: host.os.version
 - external: ecs
   name: host.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/system/data_stream/diskio/fields/agent.yml b/packages/system/data_stream/diskio/fields/agent.yml
@@ -8,6 +8,7 @@
     - name: account.id
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
 
@@ -16,12 +17,14 @@
     - name: availability_zone
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Availability zone in which this host is running.
       example: us-east-1c
     - name: instance.id
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Instance ID of the host machine.
       example: i-1234567890abcdef0
@@ -39,12 +42,14 @@
     - name: provider
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
       example: aws
     - name: region
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Region in which this host is running.
       example: us-east-1
@@ -67,6 +72,7 @@
       type: keyword
       ignore_above: 1024
       description: Unique container id.
+      dimension: true
     - name: image.name
       level: extended
       type: keyword
@@ -133,6 +139,7 @@
     - name: name
       level: core
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: 'Name of the host.
 
@@ -149,17 +156,6 @@
       ignore_above: 1024
       description: Operating system kernel version as a raw string.
       example: 4.4.0-112-generic
-    - name: os.name
-      level: extended
-      type: keyword
-      ignore_above: 1024
-      multi_fields:
-        - name: text
-          type: text
-          norms: false
-          default_field: false
-      description: Operating system name, without the version.
-      example: Mac OS X
     - name: os.platform
       level: extended
       type: keyword

diff --git a/packages/system/data_stream/diskio/fields/ecs.yml b/packages/system/data_stream/diskio/fields/ecs.yml
@@ -24,3 +24,6 @@
   name: host.os.version
 - external: ecs
   name: host.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/system/data_stream/diskio/fields/fields.yml b/packages/system/data_stream/diskio/fields/fields.yml
@@ -3,6 +3,7 @@
   fields:
     - name: name
       type: keyword
+      dimension: true
       description: |
         The disk name.
     - name: serial_number

diff --git a/packages/system/data_stream/filesystem/fields/agent.yml b/packages/system/data_stream/filesystem/fields/agent.yml
@@ -9,19 +9,22 @@
       level: extended
       type: keyword
       ignore_above: 1024
+      dimension: true
       description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
 
         Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.'
       example: 666777888999
     - name: availability_zone
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Availability zone in which this host is running.
       example: us-east-1c
     - name: instance.id
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Instance ID of the host machine.
       example: i-1234567890abcdef0
@@ -39,12 +42,14 @@
     - name: provider
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
       example: aws
     - name: region
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Region in which this host is running.
       example: us-east-1
@@ -66,6 +71,7 @@
       level: core
       type: keyword
       ignore_above: 1024
+      dimension: true
       description: Unique container id.
     - name: image.name
       level: extended
@@ -134,6 +140,7 @@
       level: core
       type: keyword
       ignore_above: 1024
+      dimension: true
       description: 'Name of the host.
 
         It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.'

diff --git a/packages/system/data_stream/filesystem/fields/ecs.yml b/packages/system/data_stream/filesystem/fields/ecs.yml
@@ -0,0 +1,3 @@
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/system/data_stream/filesystem/fields/fields.yml b/packages/system/data_stream/filesystem/fields/fields.yml
@@ -10,6 +10,7 @@
         The disk space available to an unprivileged user in bytes.
     - name: device_name
       type: keyword
+      dimension: true
       description: |
         The disk name. For example: `/dev/disk1`
     - name: type
@@ -18,6 +19,7 @@
         The disk type. For example: `ext4`
     - name: mount_point
       type: keyword
+      dimension: true
       description: |
         The mounting point. For example: `/`
     - name: files

diff --git a/packages/system/data_stream/fsstat/fields/agent.yml b/packages/system/data_stream/fsstat/fields/agent.yml
@@ -8,6 +8,7 @@
     - name: account.id
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
 
@@ -16,12 +17,14 @@
     - name: availability_zone
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Availability zone in which this host is running.
       example: us-east-1c
     - name: instance.id
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Instance ID of the host machine.
       example: i-1234567890abcdef0
@@ -39,12 +42,14 @@
     - name: provider
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
       example: aws
     - name: region
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Region in which this host is running.
       example: us-east-1
@@ -65,6 +70,7 @@
     - name: id
       level: core
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Unique container id.
     - name: image.name
@@ -133,6 +139,7 @@
     - name: name
       level: core
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: 'Name of the host.
 

diff --git a/packages/system/data_stream/fsstat/fields/ecs.yml b/packages/system/data_stream/fsstat/fields/ecs.yml
@@ -22,3 +22,6 @@
   name: host.os.version
 - external: ecs
   name: host.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/system/data_stream/load/fields/agent.yml b/packages/system/data_stream/load/fields/agent.yml
@@ -8,6 +8,7 @@
     - name: account.id
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
 
@@ -16,6 +17,7 @@
     - name: availability_zone
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Availability zone in which this host is running.
       example: us-east-1c
@@ -47,13 +49,13 @@
     - name: region
       level: extended
       type: keyword
+      dimension: true
       ignore_above: 1024
       description: Region in which this host is running.
       example: us-east-1
     - name: project.id
       type: keyword
       description: Name of the project in Google Cloud.
-      dimension: true
     - name: image.id
       type: keyword
       description: Image ID for the cloud instance.
@@ -154,17 +156,6 @@
       ignore_above: 1024
       description: Operating system kernel version as a raw string.
       example: 4.4.0-112-generic
-    - name: os.name
-      level: extended
-      type: keyword
-      ignore_above: 1024
-      multi_fields:
-        - name: text
-          type: text
-          norms: false
-          default_field: false
-      description: Operating system name, without the version.
-      example: Mac OS X
     - name: os.platform
       level: extended
       type: keyword
@@ -201,11 +192,3 @@
       description: >
         OS codename, if any.
 
-- name: agent
-  title: Agent
-  type: group
-  fields:
-    - name: id
-      type: keyword
-      ignore_above: 1024
-      dimension: true
diff --git a/packages/system/data_stream/load/fields/ecs.yml b/packages/system/data_stream/load/fields/ecs.yml
@@ -22,3 +22,6 @@
   name: host.os.version
 - external: ecs
   name: host.type
+- external: ecs
+  name: agent.id
+  dimension: true
diff --git a/packages/system/data_stream/memory/fields/agent.yml b/packages/system/data_stream/memory/fields/agent.yml
@@ -13,12 +13,14 @@
 
         Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.'
       example: 666777888999
+      dimension: true
     - name: availability_zone
       level: extended
       type: keyword
       ignore_above: 1024
       description: Availability zone in which this host is running.
       example: us-east-1c
+      dimension: true
     - name: instance.id
       level: extended
       type: keyword
@@ -50,10 +52,10 @@
       ignore_above: 1024
       description: Region in which this host is running.
       example: us-east-1
+      dimension: true
     - name: project.id
       type: keyword
       description: Name of the project in Google Cloud.
-      dimension: true
     - name: image.id
       type: keyword
       description: Image ID for the cloud instance.
@@ -201,11 +203,3 @@
       description: >
         OS codename, if any.
 
-- name: agent
-  title: Agent
-  type: group
-  fields:
-    - name: id
-      type: keyword
-      ignore_above: 1024
-      dimension: true