Add cluster name to EP/Lists telemetry (#122429) (#122534)

* Send additional http header with cluster name included for join. * Add cluster / license info to ep document. * Add cluster / license info to lists document. * Fix tests. * Be explicit with type imports. Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> (cherry picked from commit 5254cb5) Co-authored-by: Pete Hampton <pjhampton@users.noreply.github.com>
elastic · Jan 10, 2022 · 3e03c5e · 3e03c5e
1 parent 743b2b5
commit 3e03c5e
Show file tree

Hide file tree

Showing 11 changed files with 154 additions and 33 deletions.
diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts
@@ -11,9 +11,6 @@ import { TelemetryReceiver } from '../receiver';
 import { SecurityTelemetryTaskConfig } from '../task';
 import { PackagePolicy } from '../../../../../fleet/common/types/models/package_policy';
 
-/**
- * Creates a mocked Telemetry Events Sender
- */
 export const createMockTelemetryEventsSender = (
   enableTelemetry?: boolean
 ): jest.Mocked<TelemetryEventsSender> => {
@@ -37,6 +34,7 @@ export const createMockTelemetryReceiver = (
 ): jest.Mocked<TelemetryReceiver> => {
   return {
     start: jest.fn(),
+    fetchClusterInfo: jest.fn(),
     fetchLicenseInfo: jest.fn(),
     copyLicenseFields: jest.fn(),
     fetchFleetAgents: jest.fn(),
@@ -49,9 +47,6 @@ export const createMockTelemetryReceiver = (
   } as unknown as jest.Mocked<TelemetryReceiver>;
 };
 
-/**
- * Creates a mocked package policy
- */
 export const createMockPackagePolicy = (): jest.Mocked<PackagePolicy> => {
   return {
     id: jest.fn(),
@@ -65,9 +60,6 @@ export const createMockPackagePolicy = (): jest.Mocked<PackagePolicy> => {
   } as unknown as jest.Mocked<PackagePolicy>;
 };
 
-/**
- * Creates a mocked Security Telemetry Task Config
- */
 export const createMockSecurityTelemetryTask = (
   testType?: string,
   testLastTimestamp?: string
@@ -83,9 +75,6 @@ export const createMockSecurityTelemetryTask = (
   } as unknown as jest.Mocked<SecurityTelemetryTaskConfig>;
 };
 
-/**
- * Creates a mocked Task Instance
- */
 export const createMockTaskInstance = (testId: string, testType: string): ConcreteTaskInstance => {
   return {
     id: testId,

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts b/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { TelemetryEvent } from './types';
+import type { TelemetryEvent } from './types';
 
 export interface AllowlistFields {
   [key: string]: boolean | AllowlistFields;

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts
@@ -20,7 +20,7 @@ import {
   isPackagePolicyList,
   templateExceptionList,
 } from './helpers';
-import { ExceptionListItem } from './types';
+import type { ESClusterInfo, ESLicense, ExceptionListItem } from './types';
 
 describe('test diagnostic telemetry scheduled task timing helper', () => {
   test('test -5 mins is returned when there is no previous task run', async () => {
@@ -135,9 +135,20 @@ describe('test package policy type guard', () => {
 });
 
 describe('list telemetry schema', () => {
+  const clusterInfo = {
+    cluster_uuid: 'stub_cluster',
+    cluster_name: 'stub_cluster',
+  } as ESClusterInfo;
+  const licenseInfo = { uid: 'stub_license' } as ESLicense;
+
   test('detection rules document is correctly formed', () => {
     const data = [{ id: 'test_1' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_DETECTION_RULE_EXCEPTION);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_DETECTION_RULE_EXCEPTION
+    );
 
     expect(templatedItems[0]?.detection_rule).not.toBeUndefined();
     expect(templatedItems[0]?.endpoint_exception).toBeUndefined();
@@ -147,7 +158,12 @@ describe('list telemetry schema', () => {
 
   test('detection rules document is correctly formed with multiple entries', () => {
     const data = [{ id: 'test_2' }, { id: 'test_2' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_DETECTION_RULE_EXCEPTION);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_DETECTION_RULE_EXCEPTION
+    );
 
     expect(templatedItems[0]?.detection_rule).not.toBeUndefined();
     expect(templatedItems[1]?.detection_rule).not.toBeUndefined();
@@ -158,7 +174,12 @@ describe('list telemetry schema', () => {
 
   test('trusted apps document is correctly formed', () => {
     const data = [{ id: 'test_1' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_TRUSTED_APPLICATION);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_TRUSTED_APPLICATION
+    );
 
     expect(templatedItems[0]?.detection_rule).toBeUndefined();
     expect(templatedItems[0]?.endpoint_exception).toBeUndefined();
@@ -168,7 +189,12 @@ describe('list telemetry schema', () => {
 
   test('trusted apps document is correctly formed with multiple entries', () => {
     const data = [{ id: 'test_2' }, { id: 'test_2' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_TRUSTED_APPLICATION);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_TRUSTED_APPLICATION
+    );
 
     expect(templatedItems[0]?.detection_rule).toBeUndefined();
     expect(templatedItems[0]?.endpoint_exception).toBeUndefined();
@@ -179,7 +205,12 @@ describe('list telemetry schema', () => {
 
   test('endpoint exception document is correctly formed', () => {
     const data = [{ id: 'test_3' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EXCEPTION);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_ENDPOINT_EXCEPTION
+    );
 
     expect(templatedItems[0]?.detection_rule).toBeUndefined();
     expect(templatedItems[0]?.endpoint_exception).not.toBeUndefined();
@@ -189,7 +220,12 @@ describe('list telemetry schema', () => {
 
   test('endpoint exception document is correctly formed with multiple entries', () => {
     const data = [{ id: 'test_4' }, { id: 'test_4' }, { id: 'test_4' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EXCEPTION);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_ENDPOINT_EXCEPTION
+    );
 
     expect(templatedItems[0]?.detection_rule).toBeUndefined();
     expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined();
@@ -201,7 +237,12 @@ describe('list telemetry schema', () => {
 
   test('endpoint event filters document is correctly formed', () => {
     const data = [{ id: 'test_5' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EVENT_FILTER);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_ENDPOINT_EVENT_FILTER
+    );
 
     expect(templatedItems[0]?.detection_rule).toBeUndefined();
     expect(templatedItems[0]?.endpoint_event_filter).not.toBeUndefined();
@@ -211,7 +252,12 @@ describe('list telemetry schema', () => {
 
   test('endpoint event filters document is correctly formed with multiple entries', () => {
     const data = [{ id: 'test_6' }, { id: 'test_6' }] as ExceptionListItem[];
-    const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EVENT_FILTER);
+    const templatedItems = templateExceptionList(
+      data,
+      clusterInfo,
+      licenseInfo,
+      LIST_ENDPOINT_EVENT_FILTER
+    );
 
     expect(templatedItems[0]?.detection_rule).toBeUndefined();
     expect(templatedItems[0]?.endpoint_event_filter).not.toBeUndefined();

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts
@@ -9,7 +9,13 @@ import moment from 'moment';
 import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
 import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy';
 import { copyAllowlistedFields, exceptionListEventFields } from './filters';
-import { ExceptionListItem, ListTemplate, TelemetryEvent } from './types';
+import type {
+  ExceptionListItem,
+  ESClusterInfo,
+  ESLicense,
+  ListTemplate,
+  TelemetryEvent,
+} from './types';
 import {
   LIST_DETECTION_RULE_EXCEPTION,
   LIST_ENDPOINT_EXCEPTION,
@@ -160,10 +166,18 @@ export const ruleExceptionListItemToTelemetryEvent = (
  * @param listType
  * @returns lists telemetry schema
  */
-export const templateExceptionList = (listData: ExceptionListItem[], listType: string) => {
+export const templateExceptionList = (
+  listData: ExceptionListItem[],
+  clusterInfo: ESClusterInfo,
+  licenseInfo: ESLicense | undefined,
+  listType: string
+) => {
   return listData.map((item) => {
     const template: ListTemplate = {
       '@timestamp': moment().toISOString(),
+      cluster_uuid: clusterInfo.cluster_uuid,
+      cluster_name: clusterInfo.cluster_name,
+      license_id: licenseInfo?.uid,
     };
 
     // cast exception list type to a TelemetryEvent for allowlist filtering

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts
@@ -22,7 +22,7 @@ import {
   trustedApplicationToTelemetryEntry,
   ruleExceptionListItemToTelemetryEvent,
 } from './helpers';
-import {
+import type {
   TelemetryEvent,
   ESLicense,
   ESClusterInfo,
@@ -316,7 +316,7 @@ export class TelemetryReceiver {
     };
   }
 
-  private async fetchClusterInfo(): Promise<ESClusterInfo> {
+  public async fetchClusterInfo(): Promise<ESClusterInfo> {
     if (this.esClient === undefined || this.esClient === null) {
       throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation');
     }

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts
@@ -21,7 +21,7 @@ import { TelemetryReceiver } from './receiver';
 import { allowlistEventFields, copyAllowlistedFields } from './filters';
 import { createTelemetryTaskConfigs } from './tasks';
 import { createUsageCounterLabel } from './helpers';
-import { TelemetryEvent } from './types';
+import type { TelemetryEvent } from './types';
 import { TELEMETRY_MAX_BUFFER_SIZE } from './constants';
 import { SecurityTelemetryTask, SecurityTelemetryTaskConfig } from './task';
 
@@ -178,6 +178,7 @@ export class TelemetryEventsSender {
         telemetryUrl,
         'alerts-endpoint',
         clusterInfo?.cluster_uuid,
+        clusterInfo?.cluster_name,
         clusterInfo?.version?.number,
         licenseInfo?.uid
       );
@@ -220,6 +221,7 @@ export class TelemetryEventsSender {
         telemetryUrl,
         channel,
         clusterInfo?.cluster_uuid,
+        clusterInfo?.cluster_name,
         clusterInfo?.version?.number,
         licenseInfo?.uid
       );
@@ -254,6 +256,7 @@ export class TelemetryEventsSender {
     telemetryUrl: string,
     channel: string,
     clusterUuid: string | undefined,
+    clusterName: string | undefined,
     clusterVersionNumber: string | undefined,
     licenseId: string | undefined
   ) {
@@ -265,6 +268,7 @@ export class TelemetryEventsSender {
         headers: {
           'Content-Type': 'application/x-ndjson',
           'X-Elastic-Cluster-ID': clusterUuid,
+          'X-Elastic-Cluster-Name': clusterName,
           'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '8.0.0',
           ...(licenseId ? { 'X-Elastic-License-ID': licenseId } : {}),
         },

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts
@@ -10,7 +10,7 @@ import { LIST_DETECTION_RULE_EXCEPTION, TELEMETRY_CHANNEL_LISTS } from '../const
 import { batchTelemetryRecords, templateExceptionList } from '../helpers';
 import { TelemetryEventsSender } from '../sender';
 import { TelemetryReceiver } from '../receiver';
-import { ExceptionListItem, RuleSearchResult } from '../types';
+import type { ExceptionListItem, ESClusterInfo, ESLicense, RuleSearchResult } from '../types';
 import { TaskExecutionPeriod } from '../task';
 
 export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: number) {
@@ -27,6 +27,20 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n
       sender: TelemetryEventsSender,
       taskExecutionPeriod: TaskExecutionPeriod
     ) => {
+      const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([
+        receiver.fetchClusterInfo(),
+        receiver.fetchLicenseInfo(),
+      ]);
+
+      const clusterInfo =
+        clusterInfoPromise.status === 'fulfilled'
+          ? clusterInfoPromise.value
+          : ({} as ESClusterInfo);
+      const licenseInfo =
+        licenseInfoPromise.status === 'fulfilled'
+          ? licenseInfoPromise.value
+          : ({} as ESLicense | undefined);
+
       // Lists Telemetry: Detection Rules
 
       const { body: prebuiltRules } = await receiver.fetchDetectionRules();
@@ -69,6 +83,8 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n
 
       const detectionRuleExceptionsJson = templateExceptionList(
         detectionRuleExceptions,
+        clusterInfo,
+        licenseInfo,
         LIST_DETECTION_RULE_EXCEPTION
       );
 

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts
@@ -8,7 +8,7 @@
 import { Logger } from 'src/core/server';
 import { getPreviousDiagTaskTimestamp } from '../helpers';
 import { TelemetryEventsSender } from '../sender';
-import { TelemetryEvent } from '../types';
+import type { TelemetryEvent } from '../types';
 import { TelemetryReceiver } from '../receiver';
 import { TaskExecutionPeriod } from '../task';
 

diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts
@@ -7,10 +7,12 @@
 
 import { Logger } from 'src/core/server';
 import { TelemetryEventsSender } from '../sender';
-import {
+import type {
   EndpointMetricsAggregation,
   EndpointPolicyResponseAggregation,
   EndpointPolicyResponseDocument,
+  ESClusterInfo,
+  ESLicense,
 } from '../types';
 import { TelemetryReceiver } from '../receiver';
 import { TaskExecutionPeriod } from '../task';
@@ -52,6 +54,20 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) {
         throw new Error('last execution timestamp is required');
       }
 
+      const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([
+        receiver.fetchClusterInfo(),
+        receiver.fetchLicenseInfo(),
+      ]);
+
+      const clusterInfo =
+        clusterInfoPromise.status === 'fulfilled'
+          ? clusterInfoPromise.value
+          : ({} as ESClusterInfo);
+      const licenseInfo =
+        licenseInfoPromise.status === 'fulfilled'
+          ? licenseInfoPromise.value
+          : ({} as ESLicense | undefined);
+
       const endpointData = await fetchEndpointData(
         receiver,
         taskExecutionPeriod.last,
@@ -199,6 +215,9 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) {
 
           return {
             '@timestamp': taskExecutionPeriod.current,
+            cluster_uuid: clusterInfo.cluster_uuid,
+            cluster_name: clusterInfo.cluster_name,
+            license_id: licenseInfo?.uid,
             endpoint_id: endpointAgentId,
             endpoint_version: endpoint.endpoint_version,
             endpoint_package_version: policyConfig?.package?.version || null,