[EEM] Add built in definitions for core Kubernetes entities (elastic#…

…196916)

## 🍒 Summary

This PR adds the OTEL and ECS entity definition for Kubernetes. This
covers the following datasets:
- Cluster
- Service (ECS Only)
- Pod
- ReplicaSet
- Deployment
- Statefulset
- DaemonSet
- Job
- CronJob
- Node

This PR does not include Container per @roshan-elastic 

### ✅ TODO
- [X] Use correct index pattern for SemConv data
(`metrics-k8sclusterreceiver.otel-default`,
`metrics-kubeletstatsreceiver.otel-default`)
Use global IDs instead of local IDs
- [X] Add minimal list of labels to track beyond what was already added
(wildcards are not supported, example `container.image.name` for
containers to allow to find all "redis" containers)
- [ ] Test with ECS data, SemConv data and mixed data (to check if we
get duplicates, with the container definition for example).

### 🐴 Follow up EEM features 
elastic/elastic-entity-model#170 (Add
dedicated aggregation for display name and use that instead to provide a
better label than the global ID)
elastic/elastic-entity-model#193 (Add entity
type display label to allow UI to not hard code a user friendly label)

---------

Co-authored-by: Chris Cowan <chris@elastic.co>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/entity_manager/server/lib/entities/built_in/index.ts

@@ -10,10 +10,13 @@ import { builtInServicesFromEcsEntityDefinition } from './services_from_ecs_data
 import { builtInHostsFromEcsEntityDefinition } from './hosts_from_ecs_data';
 import { builtInContainersFromEcsEntityDefinition } from './containers_from_ecs_data';
 
+import * as kubernetes from './kubernetes';
+
 export { BUILT_IN_ID_PREFIX } from './constants';
 
 export const builtInDefinitions: EntityDefinition[] = [
   builtInServicesFromEcsEntityDefinition,
   builtInHostsFromEcsEntityDefinition,
   builtInContainersFromEcsEntityDefinition,
+  ...Object.values(kubernetes),
 ];

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_index_patterns.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const commonEcsIndexPatterns = ['metrics-kubernetes*', 'logs-*'];

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_metadata.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MetadataField } from '@kbn/entities-schema';
+import { globalMetadata } from './global_metadata';
+
+export const commonEcsMetadata: MetadataField[] = [
+  ...globalMetadata,
+  {
+    source: 'orchestrator.namespace',
+    destination: 'orchestrator.namespace',
+    aggregation: { type: 'terms', limit: 10 },
+  },
+  {
+    source: 'orchestrator.cluster_ip',
+    destination: 'orchestrator.cluster_id',
+    aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+  },
+  {
+    source: 'orchestrator.cluster_name',
+    destination: 'orchestrator.cluster_name',
+    aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+  },
+];

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/global_metadata.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MetadataField } from '@kbn/entities-schema';
+
+export const globalMetadata: MetadataField[] = [
+  {
+    source: '_index',
+    destination: 'source_index',
+    aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+  },
+  {
+    source: 'data_stream.type',
+    destination: 'source_data_stream.type',
+    aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+  },
+  {
+    source: 'data_stream.dataset',
+    destination: 'source_data_stream.dataset',
+    aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+  },
+];

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/otel_index_patterns.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const commonOtelIndexPatterns = ['metrics-*otel*', 'logs-*'];

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/otel_metadata.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MetadataField } from '@kbn/entities-schema';
+import { globalMetadata } from './global_metadata';
+
+export const commonOtelMetadata: MetadataField[] = [
+  ...globalMetadata,
+  {
+    source: 'k8s.namespace.name',
+    destination: 'k8s.namespace.name',
+    aggregation: { type: 'terms', limit: 10 },
+  },
+  {
+    source: 'k8s.cluster.name',
+    destination: 'k8s.cluster.name',
+    aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+  },
+];

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/cluster.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+import { globalMetadata } from '../common/global_metadata';
+
+export const builtInKubernetesClusterEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_cluster_ecs`,
+    filter: 'orchestrator.cluster.name: *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes Clusters from ECS data',
+    description:
+      'This definition extracts Kubernetes cluster entities from the Kubernetes integration data streams',
+    type: 'k8s.cluster.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['orchestrator.cluster.name'],
+    displayNameTemplate: '{{orchestrator.cluster.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: [
+      ...globalMetadata,
+      {
+        source: 'orchestrator.namespace',
+        destination: 'orchestrator.namespace',
+        aggregation: { type: 'terms', limit: 10 },
+      },
+      {
+        source: 'orchestrator.cluster_ip',
+        destination: 'orchestrator.cluster_id',
+        aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+      },
+    ],
+  });

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/cron_job.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+import { commonEcsMetadata } from '../common/ecs_metadata';
+
+export const builtInKubernetesCronJobEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_cron_job_ecs`,
+    filter: 'kubernetes.cronjob.uid : *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes CronJob from ECS data',
+    description:
+      'This definition extracts Kubernetes cron job entities from the Kubernetes integration data streams',
+    type: 'k8s.cronjob.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['kubernetes.cronjob.uid'],
+    displayNameTemplate: '{{kubernetes.cronjob.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: commonEcsMetadata,
+  });

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/daemon_set.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+import { commonEcsMetadata } from '../common/ecs_metadata';
+
+export const builtInKubernetesDaemonSetEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_daemon_set_ecs`,
+    filter: 'kubernetes.daemonset.uid : *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes DaemonSet from ECS data',
+    description:
+      'This definition extracts Kubernetes daemon set entities from the Kubernetes integration data streams',
+    type: 'k8s.daemonset.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['kubernetes.daemonset.name'],
+    displayNameTemplate: '{{kubernetes.daemonset.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: commonEcsMetadata,
+  });

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/deployment.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsMetadata } from '../common/ecs_metadata';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+
+export const builtInKubernetesDeploymentEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_deployment_ecs`,
+    filter: 'kubernetes.deployment.uid : *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes Deployment from ECS data',
+    description:
+      'This definition extracts Kubernetes deployment entities from the Kubernetes integration data streams',
+    type: 'k8s.deployment.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['kubernetes.deployment.uid'],
+    displayNameTemplate: '{{kubernetes.deployment.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: commonEcsMetadata,
+  });

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { builtInKubernetesClusterEcsEntityDefinition } from './cluster';
+export { builtInKubernetesNodeEcsEntityDefinition } from './node';
+export { builtInKubernetesPodEcsEntityDefinition } from './pod';
+export { builtInKubernetesReplicaSetEcsEntityDefinition } from './replica_set';
+export { builtInKubernetesDeploymentEcsEntityDefinition } from './deployment';
+export { builtInKubernetesStatefulSetEcsEntityDefinition } from './stateful_set';
+export { builtInKubernetesDaemonSetEcsEntityDefinition } from './daemon_set';
+export { builtInKubernetesJobEcsEntityDefinition } from './job';
+export { builtInKubernetesCronJobEcsEntityDefinition } from './cron_job';
+export { builtInKubernetesServiceEcsEntityDefinition } from './service';

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/job.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+import { commonEcsMetadata } from '../common/ecs_metadata';
+
+export const builtInKubernetesJobEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_job_ecs`,
+    filter: 'kubernetes.job.uid : *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes Job from ECS data',
+    description:
+      'This definition extracts Kubernetes job entities from the Kubernetes integration data streams',
+    type: 'k8s.job.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['kubernetes.job.uid'],
+    displayNameTemplate: '{{kubernetes.job.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: commonEcsMetadata,
+  });

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/node.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+import { commonEcsMetadata } from '../common/ecs_metadata';
+
+export const builtInKubernetesNodeEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_node_ecs`,
+    filer: 'kubernetes.node.uid : *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes Node from ECS data',
+    description:
+      'This definition extracts Kubernetes node entities from the Kubernetes integration data streams',
+    type: 'k8s.node.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['kubernetes.node.uid'],
+    displayNameTemplate: '{{kubernetes.node.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: commonEcsMetadata,
+  });

x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/pod.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsMetadata } from '../common/ecs_metadata';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+
+export const builtInKubernetesPodEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_pod_ecs`,
+    filter: 'kubernetes.pod.uid: *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes Pod from ECS data',
+    description:
+      'This definition extracts Kubernetes pod entities from the Kubernetes integration data streams',
+    type: 'k8s.pod.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['kubernetes.pod.name'],
+    displayNameTemplate: '{{kubernetes.pod.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: commonEcsMetadata,
+  });