From 3e03c5e7039898184049119299041d6ec1cd2827 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:28:59 -0500 Subject: [PATCH] Add cluster name to EP/Lists telemetry (#122429) (#122534) * Send additional http header with cluster name included for join. * Add cluster / license info to ep document. * Add cluster / license info to lists document. * Fix tests. * Be explicit with type imports. Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> (cherry picked from commit 5254cb5768c03e0df364e6577236cd0f3e008ba1) Co-authored-by: Pete Hampton --- .../server/lib/telemetry/__mocks__/index.ts | 13 +--- .../server/lib/telemetry/filters.ts | 2 +- .../server/lib/telemetry/helpers.test.ts | 64 ++++++++++++++++--- .../server/lib/telemetry/helpers.ts | 18 +++++- .../server/lib/telemetry/receiver.ts | 4 +- .../server/lib/telemetry/sender.ts | 6 +- .../lib/telemetry/tasks/detection_rule.ts | 18 +++++- .../server/lib/telemetry/tasks/diagnostic.ts | 2 +- .../server/lib/telemetry/tasks/endpoint.ts | 21 +++++- .../lib/telemetry/tasks/security_lists.ts | 36 ++++++++++- .../server/lib/telemetry/types.ts | 3 + 11 files changed, 154 insertions(+), 33 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts index b6657e7753364..4d9b67af6c311 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/__mocks__/index.ts @@ -11,9 +11,6 @@ import { TelemetryReceiver } from '../receiver'; import { SecurityTelemetryTaskConfig } from '../task'; import { PackagePolicy } from '../../../../../fleet/common/types/models/package_policy'; -/** - * Creates a mocked Telemetry Events Sender - */ export const createMockTelemetryEventsSender = ( enableTelemetry?: boolean ): jest.Mocked => { @@ -37,6 +34,7 @@ export const createMockTelemetryReceiver = ( ): jest.Mocked => { return { start: jest.fn(), + fetchClusterInfo: jest.fn(), fetchLicenseInfo: jest.fn(), copyLicenseFields: jest.fn(), fetchFleetAgents: jest.fn(), @@ -49,9 +47,6 @@ export const createMockTelemetryReceiver = ( } as unknown as jest.Mocked; }; -/** - * Creates a mocked package policy - */ export const createMockPackagePolicy = (): jest.Mocked => { return { id: jest.fn(), @@ -65,9 +60,6 @@ export const createMockPackagePolicy = (): jest.Mocked => { } as unknown as jest.Mocked; }; -/** - * Creates a mocked Security Telemetry Task Config - */ export const createMockSecurityTelemetryTask = ( testType?: string, testLastTimestamp?: string @@ -83,9 +75,6 @@ export const createMockSecurityTelemetryTask = ( } as unknown as jest.Mocked; }; -/** - * Creates a mocked Task Instance - */ export const createMockTaskInstance = (testId: string, testType: string): ConcreteTaskInstance => { return { id: testId, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts b/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts index 40377ba72547c..452717f1efb4f 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { TelemetryEvent } from './types'; +import type { TelemetryEvent } from './types'; export interface AllowlistFields { [key: string]: boolean | AllowlistFields; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts index 77c78c4dba28d..cacb34dc8fa70 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts @@ -20,7 +20,7 @@ import { isPackagePolicyList, templateExceptionList, } from './helpers'; -import { ExceptionListItem } from './types'; +import type { ESClusterInfo, ESLicense, ExceptionListItem } from './types'; describe('test diagnostic telemetry scheduled task timing helper', () => { test('test -5 mins is returned when there is no previous task run', async () => { @@ -135,9 +135,20 @@ describe('test package policy type guard', () => { }); describe('list telemetry schema', () => { + const clusterInfo = { + cluster_uuid: 'stub_cluster', + cluster_name: 'stub_cluster', + } as ESClusterInfo; + const licenseInfo = { uid: 'stub_license' } as ESLicense; + test('detection rules document is correctly formed', () => { const data = [{ id: 'test_1' }] as ExceptionListItem[]; - const templatedItems = templateExceptionList(data, LIST_DETECTION_RULE_EXCEPTION); + const templatedItems = templateExceptionList( + data, + clusterInfo, + licenseInfo, + LIST_DETECTION_RULE_EXCEPTION + ); expect(templatedItems[0]?.detection_rule).not.toBeUndefined(); expect(templatedItems[0]?.endpoint_exception).toBeUndefined(); @@ -147,7 +158,12 @@ describe('list telemetry schema', () => { test('detection rules document is correctly formed with multiple entries', () => { const data = [{ id: 'test_2' }, { id: 'test_2' }] as ExceptionListItem[]; - const templatedItems = templateExceptionList(data, LIST_DETECTION_RULE_EXCEPTION); + const templatedItems = templateExceptionList( + data, + clusterInfo, + licenseInfo, + LIST_DETECTION_RULE_EXCEPTION + ); expect(templatedItems[0]?.detection_rule).not.toBeUndefined(); expect(templatedItems[1]?.detection_rule).not.toBeUndefined(); @@ -158,7 +174,12 @@ describe('list telemetry schema', () => { test('trusted apps document is correctly formed', () => { const data = [{ id: 'test_1' }] as ExceptionListItem[]; - const templatedItems = templateExceptionList(data, LIST_TRUSTED_APPLICATION); + const templatedItems = templateExceptionList( + data, + clusterInfo, + licenseInfo, + LIST_TRUSTED_APPLICATION + ); expect(templatedItems[0]?.detection_rule).toBeUndefined(); expect(templatedItems[0]?.endpoint_exception).toBeUndefined(); @@ -168,7 +189,12 @@ describe('list telemetry schema', () => { test('trusted apps document is correctly formed with multiple entries', () => { const data = [{ id: 'test_2' }, { id: 'test_2' }] as ExceptionListItem[]; - const templatedItems = templateExceptionList(data, LIST_TRUSTED_APPLICATION); + const templatedItems = templateExceptionList( + data, + clusterInfo, + licenseInfo, + LIST_TRUSTED_APPLICATION + ); expect(templatedItems[0]?.detection_rule).toBeUndefined(); expect(templatedItems[0]?.endpoint_exception).toBeUndefined(); @@ -179,7 +205,12 @@ describe('list telemetry schema', () => { test('endpoint exception document is correctly formed', () => { const data = [{ id: 'test_3' }] as ExceptionListItem[]; - const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EXCEPTION); + const templatedItems = templateExceptionList( + data, + clusterInfo, + licenseInfo, + LIST_ENDPOINT_EXCEPTION + ); expect(templatedItems[0]?.detection_rule).toBeUndefined(); expect(templatedItems[0]?.endpoint_exception).not.toBeUndefined(); @@ -189,7 +220,12 @@ describe('list telemetry schema', () => { test('endpoint exception document is correctly formed with multiple entries', () => { const data = [{ id: 'test_4' }, { id: 'test_4' }, { id: 'test_4' }] as ExceptionListItem[]; - const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EXCEPTION); + const templatedItems = templateExceptionList( + data, + clusterInfo, + licenseInfo, + LIST_ENDPOINT_EXCEPTION + ); expect(templatedItems[0]?.detection_rule).toBeUndefined(); expect(templatedItems[0]?.endpoint_event_filter).toBeUndefined(); @@ -201,7 +237,12 @@ describe('list telemetry schema', () => { test('endpoint event filters document is correctly formed', () => { const data = [{ id: 'test_5' }] as ExceptionListItem[]; - const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EVENT_FILTER); + const templatedItems = templateExceptionList( + data, + clusterInfo, + licenseInfo, + LIST_ENDPOINT_EVENT_FILTER + ); expect(templatedItems[0]?.detection_rule).toBeUndefined(); expect(templatedItems[0]?.endpoint_event_filter).not.toBeUndefined(); @@ -211,7 +252,12 @@ describe('list telemetry schema', () => { test('endpoint event filters document is correctly formed with multiple entries', () => { const data = [{ id: 'test_6' }, { id: 'test_6' }] as ExceptionListItem[]; - const templatedItems = templateExceptionList(data, LIST_ENDPOINT_EVENT_FILTER); + const templatedItems = templateExceptionList( + data, + clusterInfo, + licenseInfo, + LIST_ENDPOINT_EVENT_FILTER + ); expect(templatedItems[0]?.detection_rule).toBeUndefined(); expect(templatedItems[0]?.endpoint_event_filter).not.toBeUndefined(); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts index 62b632d4a0803..92ea33a7b07ed 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -9,7 +9,13 @@ import moment from 'moment'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy'; import { copyAllowlistedFields, exceptionListEventFields } from './filters'; -import { ExceptionListItem, ListTemplate, TelemetryEvent } from './types'; +import type { + ExceptionListItem, + ESClusterInfo, + ESLicense, + ListTemplate, + TelemetryEvent, +} from './types'; import { LIST_DETECTION_RULE_EXCEPTION, LIST_ENDPOINT_EXCEPTION, @@ -160,10 +166,18 @@ export const ruleExceptionListItemToTelemetryEvent = ( * @param listType * @returns lists telemetry schema */ -export const templateExceptionList = (listData: ExceptionListItem[], listType: string) => { +export const templateExceptionList = ( + listData: ExceptionListItem[], + clusterInfo: ESClusterInfo, + licenseInfo: ESLicense | undefined, + listType: string +) => { return listData.map((item) => { const template: ListTemplate = { '@timestamp': moment().toISOString(), + cluster_uuid: clusterInfo.cluster_uuid, + cluster_name: clusterInfo.cluster_name, + license_id: licenseInfo?.uid, }; // cast exception list type to a TelemetryEvent for allowlist filtering diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index b4240dac738ee..421d5bea440bb 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -22,7 +22,7 @@ import { trustedApplicationToTelemetryEntry, ruleExceptionListItemToTelemetryEvent, } from './helpers'; -import { +import type { TelemetryEvent, ESLicense, ESClusterInfo, @@ -316,7 +316,7 @@ export class TelemetryReceiver { }; } - private async fetchClusterInfo(): Promise { + public async fetchClusterInfo(): Promise { if (this.esClient === undefined || this.esClient === null) { throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index bd7e0b5bee84b..fbc42acca036b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -21,7 +21,7 @@ import { TelemetryReceiver } from './receiver'; import { allowlistEventFields, copyAllowlistedFields } from './filters'; import { createTelemetryTaskConfigs } from './tasks'; import { createUsageCounterLabel } from './helpers'; -import { TelemetryEvent } from './types'; +import type { TelemetryEvent } from './types'; import { TELEMETRY_MAX_BUFFER_SIZE } from './constants'; import { SecurityTelemetryTask, SecurityTelemetryTaskConfig } from './task'; @@ -178,6 +178,7 @@ export class TelemetryEventsSender { telemetryUrl, 'alerts-endpoint', clusterInfo?.cluster_uuid, + clusterInfo?.cluster_name, clusterInfo?.version?.number, licenseInfo?.uid ); @@ -220,6 +221,7 @@ export class TelemetryEventsSender { telemetryUrl, channel, clusterInfo?.cluster_uuid, + clusterInfo?.cluster_name, clusterInfo?.version?.number, licenseInfo?.uid ); @@ -254,6 +256,7 @@ export class TelemetryEventsSender { telemetryUrl: string, channel: string, clusterUuid: string | undefined, + clusterName: string | undefined, clusterVersionNumber: string | undefined, licenseId: string | undefined ) { @@ -265,6 +268,7 @@ export class TelemetryEventsSender { headers: { 'Content-Type': 'application/x-ndjson', 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Cluster-Name': clusterName, 'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '8.0.0', ...(licenseId ? { 'X-Elastic-License-ID': licenseId } : {}), }, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts index dd4f2b60a44c9..baca71d100cf0 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts @@ -10,7 +10,7 @@ import { LIST_DETECTION_RULE_EXCEPTION, TELEMETRY_CHANNEL_LISTS } from '../const import { batchTelemetryRecords, templateExceptionList } from '../helpers'; import { TelemetryEventsSender } from '../sender'; import { TelemetryReceiver } from '../receiver'; -import { ExceptionListItem, RuleSearchResult } from '../types'; +import type { ExceptionListItem, ESClusterInfo, ESLicense, RuleSearchResult } from '../types'; import { TaskExecutionPeriod } from '../task'; export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: number) { @@ -27,6 +27,20 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n sender: TelemetryEventsSender, taskExecutionPeriod: TaskExecutionPeriod ) => { + const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ + receiver.fetchClusterInfo(), + receiver.fetchLicenseInfo(), + ]); + + const clusterInfo = + clusterInfoPromise.status === 'fulfilled' + ? clusterInfoPromise.value + : ({} as ESClusterInfo); + const licenseInfo = + licenseInfoPromise.status === 'fulfilled' + ? licenseInfoPromise.value + : ({} as ESLicense | undefined); + // Lists Telemetry: Detection Rules const { body: prebuiltRules } = await receiver.fetchDetectionRules(); @@ -69,6 +83,8 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n const detectionRuleExceptionsJson = templateExceptionList( detectionRuleExceptions, + clusterInfo, + licenseInfo, LIST_DETECTION_RULE_EXCEPTION ); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts index fba96da63ecaf..fae6172c268f4 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts @@ -8,7 +8,7 @@ import { Logger } from 'src/core/server'; import { getPreviousDiagTaskTimestamp } from '../helpers'; import { TelemetryEventsSender } from '../sender'; -import { TelemetryEvent } from '../types'; +import type { TelemetryEvent } from '../types'; import { TelemetryReceiver } from '../receiver'; import { TaskExecutionPeriod } from '../task'; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts index a06aac19d313b..78f2889ac1d58 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts @@ -7,10 +7,12 @@ import { Logger } from 'src/core/server'; import { TelemetryEventsSender } from '../sender'; -import { +import type { EndpointMetricsAggregation, EndpointPolicyResponseAggregation, EndpointPolicyResponseDocument, + ESClusterInfo, + ESLicense, } from '../types'; import { TelemetryReceiver } from '../receiver'; import { TaskExecutionPeriod } from '../task'; @@ -52,6 +54,20 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { throw new Error('last execution timestamp is required'); } + const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ + receiver.fetchClusterInfo(), + receiver.fetchLicenseInfo(), + ]); + + const clusterInfo = + clusterInfoPromise.status === 'fulfilled' + ? clusterInfoPromise.value + : ({} as ESClusterInfo); + const licenseInfo = + licenseInfoPromise.status === 'fulfilled' + ? licenseInfoPromise.value + : ({} as ESLicense | undefined); + const endpointData = await fetchEndpointData( receiver, taskExecutionPeriod.last, @@ -199,6 +215,9 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { return { '@timestamp': taskExecutionPeriod.current, + cluster_uuid: clusterInfo.cluster_uuid, + cluster_name: clusterInfo.cluster_name, + license_id: licenseInfo?.uid, endpoint_id: endpointAgentId, endpoint_version: endpoint.endpoint_version, endpoint_package_version: policyConfig?.package?.version || null, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts index 3fead5ae33be8..d27ab801197d6 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts @@ -16,6 +16,7 @@ import { LIST_TRUSTED_APPLICATION, TELEMETRY_CHANNEL_LISTS, } from '../constants'; +import type { ESClusterInfo, ESLicense } from '../types'; import { batchTelemetryRecords, templateExceptionList } from '../helpers'; import { TelemetryEventsSender } from '../sender'; import { TelemetryReceiver } from '../receiver'; @@ -37,11 +38,30 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number) ) => { let count = 0; + const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ + receiver.fetchClusterInfo(), + receiver.fetchLicenseInfo(), + ]); + + const clusterInfo = + clusterInfoPromise.status === 'fulfilled' + ? clusterInfoPromise.value + : ({} as ESClusterInfo); + const licenseInfo = + licenseInfoPromise.status === 'fulfilled' + ? licenseInfoPromise.value + : ({} as ESLicense | undefined); + // Lists Telemetry: Trusted Applications const trustedApps = await receiver.fetchTrustedApplications(); if (trustedApps?.data) { - const trustedAppsJson = templateExceptionList(trustedApps.data, LIST_TRUSTED_APPLICATION); + const trustedAppsJson = templateExceptionList( + trustedApps.data, + clusterInfo, + licenseInfo, + LIST_TRUSTED_APPLICATION + ); logger.debug(`Trusted Apps: ${trustedAppsJson}`); count += trustedAppsJson.length; @@ -54,7 +74,12 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number) const epExceptions = await receiver.fetchEndpointList(ENDPOINT_LIST_ID); if (epExceptions?.data) { - const epExceptionsJson = templateExceptionList(epExceptions.data, LIST_ENDPOINT_EXCEPTION); + const epExceptionsJson = templateExceptionList( + epExceptions.data, + clusterInfo, + licenseInfo, + LIST_ENDPOINT_EXCEPTION + ); logger.debug(`EP Exceptions: ${epExceptionsJson}`); count += epExceptionsJson.length; @@ -67,7 +92,12 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number) const epFilters = await receiver.fetchEndpointList(ENDPOINT_EVENT_FILTERS_LIST_ID); if (epFilters?.data) { - const epFiltersJson = templateExceptionList(epFilters.data, LIST_ENDPOINT_EVENT_FILTER); + const epFiltersJson = templateExceptionList( + epFilters.data, + clusterInfo, + licenseInfo, + LIST_ENDPOINT_EVENT_FILTER + ); logger.debug(`EP Event Filters: ${epFiltersJson}`); count += epFiltersJson.length; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index c65e40895de54..d7ba5fa34cf09 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -227,6 +227,9 @@ export interface ExceptionListItem { export interface ListTemplate { '@timestamp': string; + cluster_uuid: string; + cluster_name: string; + license_id: string | undefined; detection_rule?: TelemetryEvent; endpoint_exception?: TelemetryEvent; endpoint_event_filter?: TelemetryEvent;