Add cluster name to EP/Lists telemetry

x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts

@@ -11,9 +11,6 @@ import { TelemetryReceiver } from '../receiver';
 import { SecurityTelemetryTaskConfig } from '../task';
 import { PackagePolicy } from '../../../../../fleet/common/types/models/package_policy';
 
-/**
- * Creates a mocked Telemetry Events Sender
- */
 export const createMockTelemetryEventsSender = (
   enableTelemetry?: boolean
 ): jest.Mocked<TelemetryEventsSender> => {
@@ -37,6 +34,7 @@ export const createMockTelemetryReceiver = (
 ): jest.Mocked<TelemetryReceiver> => {
   return {
     start: jest.fn(),
+    fetchClusterInfo: jest.fn(),
     fetchLicenseInfo: jest.fn(),
     copyLicenseFields: jest.fn(),
     fetchFleetAgents: jest.fn(),
@@ -49,9 +47,6 @@ export const createMockTelemetryReceiver = (
   } as unknown as jest.Mocked<TelemetryReceiver>;
 };
 
-/**
- * Creates a mocked package policy
- */
 export const createMockPackagePolicy = (): jest.Mocked<PackagePolicy> => {
   return {
     id: jest.fn(),
@@ -65,9 +60,6 @@ export const createMockPackagePolicy = (): jest.Mocked<PackagePolicy> => {
   } as unknown as jest.Mocked<PackagePolicy>;
 };
 
-/**
- * Creates a mocked Security Telemetry Task Config
- */
 export const createMockSecurityTelemetryTask = (
   testType?: string,
   testLastTimestamp?: string
@@ -83,9 +75,6 @@ export const createMockSecurityTelemetryTask = (
   } as unknown as jest.Mocked<SecurityTelemetryTaskConfig>;
 };
 
-/**
- * Creates a mocked Task Instance
- */
 export const createMockTaskInstance = (testId: string, testType: string): ConcreteTaskInstance => {
   return {
     id: testId,

x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts

@@ -20,7 +20,7 @@ import {
   isPackagePolicyList,
   templateExceptionList,
 } from './helpers';
-import { ExceptionListItem } from './types';
+import { ESClusterInfo, ESLicense, ExceptionListItem } from './types';
 
 describe('test diagnostic telemetry scheduled task timing helper', () => {
   test('test -5 mins is returned when there is no previous task run', async () => {
@@ -135,9 +135,20 @@ describe('test package policy type guard', () => {
 });
 
 describe('list telemetry schema', () => {
+  const clusterInfo = {
+    cluster_uuid: 'stub_cluster',
+    cluster_name: 'stub_cluster',
+  } as ESClusterInfo;
+  const licenseInfo = { uid: 'stub_license' } as ESLicense;
+
   test('detection rules document is correctly formed', () => {
     const data = [{ id: 'test_1' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_DETECTION_RULE_EXCEPTION);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_DETECTION_RULE_EXCEPTION
+    );
 
     expect(templatedItems[0]?.detection_rule).not.toBeUndefined();
     expect(templatedItems[0]?.endpoint_exception).toBeUndefined();
@@ -147,7 +158,12 @@ describe('list telemetry schema', () => {
 
   test('detection rules document is correctly formed with multiple entries', () => {
     const data = [{ id: 'test_2' }, { id: 'test_2' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_DETECTION_RULE_EXCEPTION);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_DETECTION_RULE_EXCEPTION
+    );
 
     expect(templatedItems[0]?.detection_rule).not.toBeUndefined();
     expect(templatedItems[1]?.detection_rule).not.toBeUndefined();
@@ -158,7 +174,12 @@ describe('list telemetry schema', () => {
 
   test('trusted apps document is correctly formed', () => {
     const data = [{ id: 'test_1' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_TRUSTED_APPLICATION);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_TRUSTED_APPLICATION
+    );
 
     expect(templatedItems[0]?.detection_rule).toBeUndefined();
     expect(templatedItems[0]?.endpoint_exception).toBeUndefined();
@@ -168,7 +189,12 @@ describe('list telemetry schema', () => {
 
   test('trusted apps document is correctly formed with multiple entries', () => {
     const data = [{ id: 'test_2' }, { id: 'test_2' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_TRUSTED_APPLICATION);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_TRUSTED_APPLICATION
+    );
 
     expect(templatedItems[0]?.detection_rule).toBeUndefined();
     expect(templatedItems[0]?.endpoint_exception).toBeUndefined();
@@ -179,7 +205,12 @@ describe('list telemetry schema', () => {
 
   test('endpoint exception document is correctly formed', () => {
     const data = [{ id: 'test_3' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EXCEPTION);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_ENDPOINT_EXCEPTION
+    );
 
     expect(templatedItems[0]?.detection_rule).toBeUndefined();
     expect(templatedItems[0]?.endpoint_exception).not.toBeUndefined();
@@ -189,7 +220,12 @@ describe('list telemetry schema', () => {
 
   test('endpoint exception document is correctly formed with multiple entries', () => {
     const data = [{ id: 'test_4' }, { id: 'test_4' }, { id: 'test_4' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EXCEPTION);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_ENDPOINT_EXCEPTION
+    );
 
     expect(templatedItems[0]?.detection_rule).toBeUndefined();
     expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined();
@@ -201,7 +237,12 @@ describe('list telemetry schema', () => {
 
   test('endpoint event filters document is correctly formed', () => {
     const data = [{ id: 'test_5' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EVENT_FILTER);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_ENDPOINT_EVENT_FILTER
+    );
 
     expect(templatedItems[0]?.detection_rule).toBeUndefined();
     expect(templatedItems[0]?.endpoint_event_filter).not.toBeUndefined();
@@ -211,7 +252,12 @@ describe('list telemetry schema', () => {
 
   test('endpoint event filters document is correctly formed with multiple entries', () => {
     const data = [{ id: 'test_6' }, { id: 'test_6' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EVENT_FILTER);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_ENDPOINT_EVENT_FILTER
+    );
 
     expect(templatedItems[0]?.detection_rule).toBeUndefined();
     expect(templatedItems[0]?.endpoint_event_filter).not.toBeUndefined();

x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts

@@ -9,7 +9,7 @@ import moment from 'moment';
 import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
 import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy';
 import { copyAllowlistedFields, exceptionListEventFields } from './filters';
-import { ExceptionListItem, ListTemplate, TelemetryEvent } from './types';
+import { ExceptionListItem, ESClusterInfo, ESLicense, ListTemplate, TelemetryEvent } from './types';
 import {
   LIST_DETECTION_RULE_EXCEPTION,
   LIST_ENDPOINT_EXCEPTION,
@@ -160,10 +160,18 @@ export const ruleExceptionListItemToTelemetryEvent = (
  * @param listType
  * @returns lists telemetry schema
  */
-export const templateExceptionList = (listData: ExceptionListItem[], listType: string) => {
+export const templateExceptionList = (
+  listData: ExceptionListItem[],
+  clusterInfo: ESClusterInfo,
+  licenseInfo: ESLicense | undefined,
+  listType: string
+) => {
   return listData.map((item) => {
     const template: ListTemplate = {
       '@timestamp': moment().toISOString(),
+      cluster_uuid: clusterInfo.cluster_uuid,
+      cluster_name: clusterInfo.cluster_name,
+      license_id: licenseInfo?.uid,
     };
 
     // cast exception list type to a TelemetryEvent for allowlist filtering

x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts

@@ -316,7 +316,7 @@ export class TelemetryReceiver {
     };
   }
 
-  private async fetchClusterInfo(): Promise<ESClusterInfo> {
+  public async fetchClusterInfo(): Promise<ESClusterInfo> {
     if (this.esClient === undefined || this.esClient === null) {
       throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation');
     }

x-pack/plugins/security_solution/server/lib/telemetry/sender.ts

@@ -178,6 +178,7 @@ export class TelemetryEventsSender {
         telemetryUrl,
         'alerts-endpoint',
         clusterInfo?.cluster_uuid,
+        clusterInfo?.cluster_name,
         clusterInfo?.version?.number,
         licenseInfo?.uid
       );
@@ -220,6 +221,7 @@ export class TelemetryEventsSender {
         telemetryUrl,
         channel,
         clusterInfo?.cluster_uuid,
+        clusterInfo?.cluster_name,
         clusterInfo?.version?.number,
         licenseInfo?.uid
       );
@@ -254,6 +256,7 @@ export class TelemetryEventsSender {
     telemetryUrl: string,
     channel: string,
     clusterUuid: string | undefined,
+    clusterName: string | undefined,
     clusterVersionNumber: string | undefined,
     licenseId: string | undefined
   ) {
@@ -265,6 +268,7 @@ export class TelemetryEventsSender {
         headers: {
           'Content-Type': 'application/x-ndjson',
           'X-Elastic-Cluster-ID': clusterUuid,
+          'X-Elastic-Cluster-Name': clusterName,
           'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '8.0.0',
           ...(licenseId ? { 'X-Elastic-License-ID': licenseId } : {}),
         },

x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts

@@ -10,7 +10,7 @@ import { LIST_DETECTION_RULE_EXCEPTION, TELEMETRY_CHANNEL_LISTS } from '../const
 import { batchTelemetryRecords, templateExceptionList } from '../helpers';
 import { TelemetryEventsSender } from '../sender';
 import { TelemetryReceiver } from '../receiver';
-import { ExceptionListItem, RuleSearchResult } from '../types';
+import { ExceptionListItem, ESClusterInfo, ESLicense, RuleSearchResult } from '../types';
 import { TaskExecutionPeriod } from '../task';
 
 export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: number) {
@@ -27,6 +27,20 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n
       sender: TelemetryEventsSender,
       taskExecutionPeriod: TaskExecutionPeriod
     ) => {
+      const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([
+        receiver.fetchClusterInfo(),
+        receiver.fetchLicenseInfo(),
+      ]);
+
+      const clusterInfo =
+        clusterInfoPromise.status === 'fulfilled'
+          ? clusterInfoPromise.value
+          : ({} as ESClusterInfo);
+      const licenseInfo =
+        licenseInfoPromise.status === 'fulfilled'
+          ? licenseInfoPromise.value
+          : ({} as ESLicense | undefined);
+
       // Lists Telemetry: Detection Rules
 
       const { body: prebuiltRules } = await receiver.fetchDetectionRules();
@@ -69,6 +83,8 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n
 
       const detectionRuleExceptionsJson = templateExceptionList(
         detectionRuleExceptions,
+        clusterInfo,
+        licenseInfo,
         LIST_DETECTION_RULE_EXCEPTION
       );
 

x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts

@@ -11,6 +11,8 @@ import {
   EndpointMetricsAggregation,
   EndpointPolicyResponseAggregation,
   EndpointPolicyResponseDocument,
+  ESClusterInfo,
+  ESLicense,
 } from '../types';
 import { TelemetryReceiver } from '../receiver';
 import { TaskExecutionPeriod } from '../task';
@@ -52,6 +54,20 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) {
         throw new Error('last execution timestamp is required');
       }
 
+      const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([
+        receiver.fetchClusterInfo(),
+        receiver.fetchLicenseInfo(),
+      ]);
+
+      const clusterInfo =
+        clusterInfoPromise.status === 'fulfilled'
+          ? clusterInfoPromise.value
+          : ({} as ESClusterInfo);
+      const licenseInfo =
+        licenseInfoPromise.status === 'fulfilled'
+          ? licenseInfoPromise.value
+          : ({} as ESLicense | undefined);
+
       const endpointData = await fetchEndpointData(
         receiver,
         taskExecutionPeriod.last,
@@ -199,6 +215,9 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) {
 
           return {
             '@timestamp': taskExecutionPeriod.current,
+            cluster_uuid: clusterInfo.cluster_uuid,
+            cluster_name: clusterInfo.cluster_name,
+            license_id: licenseInfo?.uid,
             endpoint_id: endpointAgentId,
             endpoint_version: endpoint.endpoint_version,
             endpoint_package_version: policyConfig?.package?.version || null,

x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts

@@ -16,6 +16,7 @@ import {
   LIST_TRUSTED_APPLICATION,
   TELEMETRY_CHANNEL_LISTS,
 } from '../constants';
+import { ESClusterInfo, ESLicense } from '../types';
 import { batchTelemetryRecords, templateExceptionList } from '../helpers';
 import { TelemetryEventsSender } from '../sender';
 import { TelemetryReceiver } from '../receiver';
@@ -37,11 +38,30 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number)
     ) => {
       let count = 0;
 
+      const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([
+        receiver.fetchClusterInfo(),
+        receiver.fetchLicenseInfo(),
+      ]);
+
+      const clusterInfo =
+        clusterInfoPromise.status === 'fulfilled'
+          ? clusterInfoPromise.value
+          : ({} as ESClusterInfo);
+      const licenseInfo =
+        licenseInfoPromise.status === 'fulfilled'
+          ? licenseInfoPromise.value
+          : ({} as ESLicense | undefined);
+
       // Lists Telemetry: Trusted Applications
 
       const trustedApps = await receiver.fetchTrustedApplications();
       if (trustedApps?.data) {
-        const trustedAppsJson = templateExceptionList(trustedApps.data, LIST_TRUSTED_APPLICATION);
+        const trustedAppsJson = templateExceptionList(
+          trustedApps.data,
+          clusterInfo,
+          licenseInfo,
+          LIST_TRUSTED_APPLICATION
+        );
         logger.debug(`Trusted Apps: ${trustedAppsJson}`);
         count += trustedAppsJson.length;
 
@@ -54,7 +74,12 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number)
 
       const epExceptions = await receiver.fetchEndpointList(ENDPOINT_LIST_ID);
       if (epExceptions?.data) {
-        const epExceptionsJson = templateExceptionList(epExceptions.data, LIST_ENDPOINT_EXCEPTION);
+        const epExceptionsJson = templateExceptionList(
+          epExceptions.data,
+          clusterInfo,
+          licenseInfo,
+          LIST_ENDPOINT_EXCEPTION
+        );
         logger.debug(`EP Exceptions: ${epExceptionsJson}`);
         count += epExceptionsJson.length;
 
@@ -67,7 +92,12 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number)
 
       const epFilters = await receiver.fetchEndpointList(ENDPOINT_EVENT_FILTERS_LIST_ID);
       if (epFilters?.data) {
-        const epFiltersJson = templateExceptionList(epFilters.data, LIST_ENDPOINT_EVENT_FILTER);
+        const epFiltersJson = templateExceptionList(
+          epFilters.data,
+          clusterInfo,
+          licenseInfo,
+          LIST_ENDPOINT_EVENT_FILTER
+        );
         logger.debug(`EP Event Filters: ${epFiltersJson}`);
         count += epFiltersJson.length;