[8.x] [Security Solution][Entity Analytics] Scoping the entity store …

…to spaces (#193303) (#193697)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Security Solution][Entity Analytics] Scoping the entity store to
spaces (#193303)](#193303)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Tiago Vila
Verde","email":"tiago.vilaverde@elastic.co"},"sourceCommit":{"committedDate":"2024-09-23T09:47:48Z","message":"[Security
Solution][Entity Analytics] Scoping the entity store to spaces
(#193303)\n\n## Summary\r\n\r\nThis PR introduces Kibana Spaces support
for the Entity Store.\r\nIt implements
https://github.com/elastic/security-team/issues/10530\r\n\r\n\r\n\r\n\r\n###
How to test\r\n\r\n1. Add some host/user data\r\n* Easiest is to
use\r\n[elastic/security-data-generator](https://github.com/elastic/security-documents-generator)\r\n2.
Make sure to add `entityStoreEnabled`
under\r\n`xpack.securitySolution.enableExperimental` in your
`kibana.dev.yml`\r\n3. Make sure to create a second space other than
`default`, either via\r\nthe UI or the spaces API.\r\n4. In the default
space kibana dev tools, call the
`POST\r\nkbn:/api/entity_store/engines/{entity_type}/init {}` route for
either\r\n`user` or `host`.\r\n5. Switch to the other space and call
`INIT` again.\r\n6. Check that calling the `GET
kbn:api/entity_store/engines` route in\r\neach space returns only one
engine.\r\n7. Check that calling
`GET\r\n/.kibana*/_search?q=type:entity-engine-status` returns 2
engines, one in\r\neach space.\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"16dcfa84c8e54825bd24a89697bb715012791284","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-minor","Theme:
entity_analytics","Feature:Entity Analytics","Team:Entity
Analytics","8.16
candidate"],"number":193303,"url":"https://github.com/elastic/kibana/pull/193303","mergeCommit":{"message":"[Security
Solution][Entity Analytics] Scoping the entity store to spaces
(#193303)\n\n## Summary\r\n\r\nThis PR introduces Kibana Spaces support
for the Entity Store.\r\nIt implements
https://github.com/elastic/security-team/issues/10530\r\n\r\n\r\n\r\n\r\n###
How to test\r\n\r\n1. Add some host/user data\r\n* Easiest is to
use\r\n[elastic/security-data-generator](https://github.com/elastic/security-documents-generator)\r\n2.
Make sure to add `entityStoreEnabled`
under\r\n`xpack.securitySolution.enableExperimental` in your
`kibana.dev.yml`\r\n3. Make sure to create a second space other than
`default`, either via\r\nthe UI or the spaces API.\r\n4. In the default
space kibana dev tools, call the
`POST\r\nkbn:/api/entity_store/engines/{entity_type}/init {}` route for
either\r\n`user` or `host`.\r\n5. Switch to the other space and call
`INIT` again.\r\n6. Check that calling the `GET
kbn:api/entity_store/engines` route in\r\neach space returns only one
engine.\r\n7. Check that calling
`GET\r\n/.kibana*/_search?q=type:entity-engine-status` returns 2
engines, one in\r\neach space.\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"16dcfa84c8e54825bd24a89697bb715012791284"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/193303","number":193303,"mergeCommit":{"message":"[Security
Solution][Entity Analytics] Scoping the entity store to spaces
(#193303)\n\n## Summary\r\n\r\nThis PR introduces Kibana Spaces support
for the Entity Store.\r\nIt implements
https://github.com/elastic/security-team/issues/10530\r\n\r\n\r\n\r\n\r\n###
How to test\r\n\r\n1. Add some host/user data\r\n* Easiest is to
use\r\n[elastic/security-data-generator](https://github.com/elastic/security-documents-generator)\r\n2.
Make sure to add `entityStoreEnabled`
under\r\n`xpack.securitySolution.enableExperimental` in your
`kibana.dev.yml`\r\n3. Make sure to create a second space other than
`default`, either via\r\nthe UI or the spaces API.\r\n4. In the default
space kibana dev tools, call the
`POST\r\nkbn:/api/entity_store/engines/{entity_type}/init {}` route for
either\r\n`user` or `host`.\r\n5. Switch to the other space and call
`INIT` again.\r\n6. Check that calling the `GET
kbn:api/entity_store/engines` route in\r\neach space returns only one
engine.\r\n7. Check that calling
`GET\r\n/.kibana*/_search?q=type:entity-engine-status` returns 2
engines, one in\r\neach space.\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"16dcfa84c8e54825bd24a89697bb715012791284"}}]}]
BACKPORT-->

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts

@@ -7,11 +7,11 @@
 
 import { entityDefinitionSchema, type EntityDefinition } from '@kbn/entities-schema';
 import { ENTITY_STORE_DEFAULT_SOURCE_INDICES } from './constants';
-import { getEntityDefinitionId } from './utils/utils';
+import { buildEntityDefinitionId } from './utils/utils';
 
-export const buildHostEntityDefinition = (): EntityDefinition =>
+export const buildHostEntityDefinition = (space: string): EntityDefinition =>
   entityDefinitionSchema.parse({
-    id: getEntityDefinitionId('host'),
+    id: buildEntityDefinitionId('host', space),
     name: 'EA Host Store',
     type: 'host',
     indexPatterns: ENTITY_STORE_DEFAULT_SOURCE_INDICES,
@@ -34,22 +34,14 @@ export const buildHostEntityDefinition = (): EntityDefinition =>
     version: '1.0.0',
   });
 
-export const buildUserEntityDefinition = (): EntityDefinition =>
+export const buildUserEntityDefinition = (space: string): EntityDefinition =>
   entityDefinitionSchema.parse({
-    id: getEntityDefinitionId('user'),
+    id: buildEntityDefinitionId('user', space),
     name: 'EA User Store',
     indexPatterns: ENTITY_STORE_DEFAULT_SOURCE_INDICES,
     identityFields: ['user.name'],
     displayNameTemplate: '{{user.name}}',
-    metadata: [
-      'user.domain',
-      'user.email',
-      'user.full_name',
-      'user.hash',
-      'user.id',
-      'user.name',
-      'user.roles',
-    ],
+    metadata: ['user.email', 'user.full_name', 'user.hash', 'user.id', 'user.name', 'user.roles'],
     history: {
       timestampField: '@timestamp',
       interval: '1m',

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts

@@ -68,8 +68,8 @@ describe('EntityStoreDataClient', () => {
       expect(esClientMock.search).toHaveBeenCalledWith(
         expect.objectContaining({
           index: [
-            '.entities.v1.latest.ea_host_entity_store',
-            '.entities.v1.latest.ea_user_entity_store',
+            '.entities.v1.latest.ea_default_host_entity_store',
+            '.entities.v1.latest.ea_default_user_entity_store',
           ],
         })
       );

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts

@@ -15,12 +15,12 @@ import type {
   InitEntityStoreRequestBody,
   InitEntityStoreResponse,
 } from '../../../../common/api/entity_analytics/entity_store/engine/init.gen';
+
 import type {
-  EngineDescriptor,
   EntityType,
   InspectQuery,
 } from '../../../../common/api/entity_analytics/entity_store/common.gen';
-import { entityEngineDescriptorTypeName } from './saved_object';
+
 import { EngineDescriptorClient } from './saved_object/engine_descriptor';
 import { getEntitiesIndexName, getEntityDefinition } from './utils/utils';
 import { ENGINE_STATUS, MAX_SEARCH_RESPONSE_SIZE } from './constants';
@@ -45,14 +45,17 @@ interface SearchEntitiesParams {
 export class EntityStoreDataClient {
   private engineClient: EngineDescriptorClient;
   constructor(private readonly options: EntityStoreClientOpts) {
-    this.engineClient = new EngineDescriptorClient(options.soClient);
+    this.engineClient = new EngineDescriptorClient({
+      soClient: options.soClient,
+      namespace: options.namespace,
+    });
   }
 
   public async init(
     entityType: EntityType,
     { indexPattern = '', filter = '' }: InitEntityStoreRequestBody
   ): Promise<InitEntityStoreResponse> {
-    const definition = getEntityDefinition(entityType);
+    const definition = getEntityDefinition(entityType, this.options.namespace);
 
     this.options.logger.info(`Initializing entity store for ${entityType}`);
 
@@ -72,7 +75,7 @@ export class EntityStoreDataClient {
   }
 
   public async start(entityType: EntityType) {
-    const definition = getEntityDefinition(entityType);
+    const definition = getEntityDefinition(entityType, this.options.namespace);
 
     const descriptor = await this.engineClient.get(entityType);
 
@@ -89,7 +92,7 @@ export class EntityStoreDataClient {
   }
 
   public async stop(entityType: EntityType) {
-    const definition = getEntityDefinition(entityType);
+    const definition = getEntityDefinition(entityType, this.options.namespace);
 
     const descriptor = await this.engineClient.get(entityType);
 
@@ -110,18 +113,11 @@ export class EntityStoreDataClient {
   }
 
   public async list() {
-    return this.options.soClient
-      .find<EngineDescriptor>({
-        type: entityEngineDescriptorTypeName,
-      })
-      .then(({ saved_objects: engines }) => ({
-        engines: engines.map((engine) => engine.attributes),
-        count: engines.length,
-      }));
+    return this.engineClient.list();
   }
 
   public async delete(entityType: EntityType, deleteData: boolean) {
-    const { id } = getEntityDefinition(entityType);
+    const { id } = getEntityDefinition(entityType, this.options.namespace);
 
     this.options.logger.info(`Deleting entity store for ${entityType}`);
 
@@ -138,7 +134,7 @@ export class EntityStoreDataClient {
   }> {
     const { page, perPage, sortField, sortOrder, filterQuery, entityTypes } = params;
 
-    const index = entityTypes.map(getEntitiesIndexName);
+    const index = entityTypes.map((type) => getEntitiesIndexName(type, this.options.namespace));
     const from = (page - 1) * perPage;
     const sort = sortField ? [{ [sortField]: sortOrder }] : undefined;
 

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/saved_object/engine_descriptor.ts

@@ -20,16 +20,21 @@ import { entityEngineDescriptorTypeName } from './engine_descriptor_type';
 import { getByEntityTypeQuery, getEntityDefinition } from '../utils/utils';
 import { ENGINE_STATUS } from '../constants';
 
+interface EngineDescriptorDependencies {
+  soClient: SavedObjectsClientContract;
+  namespace: string;
+}
+
 export class EngineDescriptorClient {
-  constructor(private readonly soClient: SavedObjectsClientContract) {}
+  constructor(private readonly deps: EngineDescriptorDependencies) {}
 
   async init(entityType: EntityType, definition: EntityDefinition, filter: string) {
     const engineDescriptor = await this.find(entityType);
 
     if (engineDescriptor.total > 0)
       throw new Error(`Entity engine for ${entityType} already exists`);
 
-    const { attributes } = await this.soClient.create<EngineDescriptor>(
+    const { attributes } = await this.deps.soClient.create<EngineDescriptor>(
       entityEngineDescriptorTypeName,
       {
         status: ENGINE_STATUS.INSTALLING,
@@ -43,7 +48,7 @@ export class EngineDescriptorClient {
   }
 
   async update(id: string, status: EngineStatus) {
-    const { attributes } = await this.soClient.update<EngineDescriptor>(
+    const { attributes } = await this.deps.soClient.update<EngineDescriptor>(
       entityEngineDescriptorTypeName,
       id,
       { status },
@@ -53,24 +58,37 @@ export class EngineDescriptorClient {
   }
 
   async find(entityType: EntityType): Promise<SavedObjectsFindResponse<EngineDescriptor>> {
-    return this.soClient.find<EngineDescriptor>({
+    return this.deps.soClient.find<EngineDescriptor>({
       type: entityEngineDescriptorTypeName,
       filter: getByEntityTypeQuery(entityType),
+      namespaces: [this.deps.namespace],
     });
   }
 
   async get(entityType: EntityType): Promise<EngineDescriptor> {
-    const { id } = getEntityDefinition(entityType);
+    const { id } = getEntityDefinition(entityType, this.deps.namespace);
 
-    const { attributes } = await this.soClient.get<EngineDescriptor>(
+    const { attributes } = await this.deps.soClient.get<EngineDescriptor>(
       entityEngineDescriptorTypeName,
       id
     );
 
     return attributes;
   }
 
+  async list() {
+    return this.deps.soClient
+      .find<EngineDescriptor>({
+        type: entityEngineDescriptorTypeName,
+        namespaces: [this.deps.namespace],
+      })
+      .then(({ saved_objects: engines }) => ({
+        engines: engines.map((engine) => engine.attributes),
+        count: engines.length,
+      }));
+  }
+
   async delete(id: string) {
-    return this.soClient.delete(entityEngineDescriptorTypeName, id);
+    return this.deps.soClient.delete(entityEngineDescriptorTypeName, id);
   }
 }

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/utils.ts

@@ -5,44 +5,34 @@
  * 2.0.
  */
 
-import type { SavedObjectsFindResponse } from '@kbn/core-saved-objects-api-server';
-
 import {
   ENTITY_LATEST,
   ENTITY_SCHEMA_VERSION_V1,
   entitiesIndexPattern,
 } from '@kbn/entities-schema';
-import type {
-  EngineDescriptor,
-  EntityType,
-} from '../../../../../common/api/entity_analytics/entity_store/common.gen';
+import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
 import { buildHostEntityDefinition, buildUserEntityDefinition } from '../definition';
+
 import { entityEngineDescriptorTypeName } from '../saved_object';
 
-export const getEntityDefinition = (entityType: EntityType) => {
-  if (entityType === 'host') return buildHostEntityDefinition();
-  if (entityType === 'user') return buildUserEntityDefinition();
+export const getEntityDefinition = (entityType: EntityType, space: string) => {
+  if (entityType === 'host') return buildHostEntityDefinition(space);
+  if (entityType === 'user') return buildUserEntityDefinition(space);
 
   throw new Error(`Unsupported entity type: ${entityType}`);
 };
 
-export const ensureEngineExists =
-  (entityType: EntityType) => (results: SavedObjectsFindResponse<EngineDescriptor>) => {
-    if (results.total === 0) {
-      throw new Error(`Entity engine for ${entityType} does not exist`);
-    }
-    return results.saved_objects[0].attributes;
-  };
-
 export const getByEntityTypeQuery = (entityType: EntityType) => {
   return `${entityEngineDescriptorTypeName}.attributes.type: ${entityType}`;
 };
 
-export const getEntitiesIndexName = (entityType: EntityType) =>
+export const getEntitiesIndexName = (entityType: EntityType, namespace: string) =>
   entitiesIndexPattern({
     schemaVersion: ENTITY_SCHEMA_VERSION_V1,
     dataset: ENTITY_LATEST,
-    definitionId: getEntityDefinitionId(entityType),
+    definitionId: buildEntityDefinitionId(entityType, namespace),
   });
 
-export const getEntityDefinitionId = (entityType: EntityType) => `ea_${entityType}_entity_store`;
+export const buildEntityDefinitionId = (entityType: EntityType, space: string) => {
+  return `ea_${space}_${entityType}_entity_store`;
+};

x-pack/test/functional/es_archives/security_solution/entity_store/data.json

@@ -2,7 +2,7 @@
   "type": "doc",
   "value": {
     "id": "a4cf452c1e0375c3d4412cb550ad1783358468a3b3b777da4829d72c7d6fb74f",
-    "index": ".entities.v1.latest.ea_user_entity_store",
+    "index": ".entities.v1.latest.ea_default_user_entity_store",
     "source": {
       "event": {
         "ingested": "2024-09-11T11:26:49.706875Z"
@@ -27,7 +27,7 @@
         "id": "LBQAgKHGmpup0Kg9nlKmeQ==",
         "type": "node",
         "firstSeenTimestamp": "2024-09-11T10:46:00.000Z",
-        "definitionId": "ea_user_entity_store"
+        "definitionId": "ea_default_user_entity_store"
       }
     }
   }
@@ -37,7 +37,7 @@
   "type": "doc",
   "value": {
     "id": "a2cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb71f",
-    "index": ".entities.v1.latest.ea_host_entity_store",
+    "index": ".entities.v1.latest.ea_default_host_entity_store",
     "source": {
       "event": {
         "ingested": "2024-09-11T11:26:49.641707Z"
@@ -78,7 +78,7 @@
         "id": "ZXKm6GEcUJY6NHkMgPPmGQ==",
         "type": "node",
         "firstSeenTimestamp": "2024-09-11T10:46:00.000Z",
-        "definitionId": "ea_host_entity_store"
+        "definitionId": "ea_default_host_entity_store"
       }
     }
   }

x-pack/test/functional/es_archives/security_solution/entity_store/mappings.json

@@ -1,7 +1,7 @@
 {
   "type": "index",
   "value": {
-    "index": ".entities.v1.latest.ea_host_entity_store",
+    "index": ".entities.v1.latest.ea_default_host_entity_store",
     "mappings": {
       "date_detection": false,
       "dynamic_templates": [
@@ -162,7 +162,7 @@
 {
   "type": "index",
   "value": {
-    "index": ".entities.v1.latest.ea_user_entity_store",
+    "index": ".entities.v1.latest.ea_default_user_entity_store",
     "mappings": {
       "date_detection": false,
       "dynamic_templates": [