From faed2f6fe10d74273f9ff3a023fabd1c9fee849d Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Fri, 30 Jul 2021 11:04:46 -0400 Subject: [PATCH 1/6] [ML] Anomaly Detection: ability to clear warning notification from jobs list - improvements (#106879) * add isClearable function for checking notifications index * clear messages from multiple indices when messages stored in multiple indices * only return clearable indices. update disable clear button check * add unit test --- .../ml/common/constants/index_patterns.ts | 1 - .../job_details/job_messages_pane.tsx | 12 +++-- .../services/ml_api_service/jobs.ts | 6 +-- .../job_audit_messages/is_clearable.test.ts | 35 +++++++++++++ .../job_audit_messages.d.ts | 10 +++- .../job_audit_messages/job_audit_messages.js | 52 ++++++++++++------- .../ml/server/routes/job_audit_messages.ts | 4 +- .../schemas/job_audit_messages_schema.ts | 1 + 8 files changed, 91 insertions(+), 30 deletions(-) create mode 100644 x-pack/plugins/ml/server/models/job_audit_messages/is_clearable.test.ts diff --git a/x-pack/plugins/ml/common/constants/index_patterns.ts b/x-pack/plugins/ml/common/constants/index_patterns.ts index cec692217546d..d7d6c343e282b 100644 --- a/x-pack/plugins/ml/common/constants/index_patterns.ts +++ b/x-pack/plugins/ml/common/constants/index_patterns.ts @@ -11,4 +11,3 @@ export const ML_ANNOTATIONS_INDEX_PATTERN = '.ml-annotations-6'; export const ML_RESULTS_INDEX_PATTERN = '.ml-anomalies-*'; export const ML_NOTIFICATION_INDEX_PATTERN = '.ml-notifications*'; -export const ML_NOTIFICATION_INDEX_02 = '.ml-notifications-000002'; diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_messages_pane.tsx b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_messages_pane.tsx index 9a4d6036428f8..92662f409d0f3 100644 --- a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_messages_pane.tsx +++ b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_messages_pane.tsx @@ -30,6 +30,7 @@ export const JobMessagesPane: FC = React.memo( const canCreateJob = checkPermission('canCreateJob'); const [messages, setMessages] = useState([]); + const [notificationIndices, setNotificationIndices] = useState([]); const [isLoading, setIsLoading] = useState(false); const [errorMessage, setErrorMessage] = useState(''); const [isClearing, setIsClearing] = useState(false); @@ -42,7 +43,10 @@ export const JobMessagesPane: FC = React.memo( const fetchMessages = async () => { setIsLoading(true); try { - setMessages(await ml.jobs.jobAuditMessages({ jobId, start, end })); + const messagesResp = await ml.jobs.jobAuditMessages({ jobId, start, end }); + + setMessages(messagesResp.messages); + setNotificationIndices(messagesResp.notificationIndices); setIsLoading(false); } catch (error) { setIsLoading(false); @@ -63,7 +67,7 @@ export const JobMessagesPane: FC = React.memo( const clearMessages = useCallback(async () => { setIsClearing(true); try { - await clearJobAuditMessages(jobId); + await clearJobAuditMessages(jobId, notificationIndices); setIsClearing(false); if (typeof refreshJobList === 'function') { refreshJobList(); @@ -77,13 +81,13 @@ export const JobMessagesPane: FC = React.memo( }) ); } - }, [jobId]); + }, [jobId, JSON.stringify(notificationIndices)]); useEffect(() => { fetchMessages(); }, []); - const disabled = messages.length > 0 && messages[0].clearable === false; + const disabled = notificationIndices.length === 0; const clearButton = ( ({ ...(start !== undefined && end !== undefined ? { start, end } : {}), }; - return httpService.http({ + return httpService.http<{ messages: JobMessage[]; notificationIndices: string[] }>({ path: `${ML_BASE_PATH}/job_audit_messages/messages${jobIdString}`, method: 'GET', query, }); }, - clearJobAuditMessages(jobId: string) { - const body = JSON.stringify({ jobId }); + clearJobAuditMessages(jobId: string, notificationIndices: string[]) { + const body = JSON.stringify({ jobId, notificationIndices }); return httpService.http<{ success: boolean; latest_cleared: number }>({ path: `${ML_BASE_PATH}/job_audit_messages/clear_messages`, method: 'PUT', diff --git a/x-pack/plugins/ml/server/models/job_audit_messages/is_clearable.test.ts b/x-pack/plugins/ml/server/models/job_audit_messages/is_clearable.test.ts new file mode 100644 index 0000000000000..8b84bfeb888b2 --- /dev/null +++ b/x-pack/plugins/ml/server/models/job_audit_messages/is_clearable.test.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isClearable } from './job_audit_messages'; + +const supportedNotificationIndices = [ + '.ml-notifications-000002', + '.ml-notifications-000003', + '.ml-notifications-000004', +]; + +const unsupportedIndices = ['.ml-notifications-000001', 'index-does-not-exist']; + +describe('jobAuditMessages - isClearable', () => { + it('should return true for indices ending in a six digit number with the last number >= 2', () => { + supportedNotificationIndices.forEach((index) => { + expect(isClearable(index)).toEqual(true); + }); + }); + + it('should return false for indices not ending in a six digit number with the last number >= 2', () => { + unsupportedIndices.forEach((index) => { + expect(isClearable(index)).toEqual(false); + }); + }); + + it('should return false for empty string or missing argument', () => { + expect(isClearable('')).toEqual(false); + expect(isClearable()).toEqual(false); + }); +}); diff --git a/x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.d.ts b/x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.d.ts index 60ea866978f1a..d3748163957db 100644 --- a/x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.d.ts +++ b/x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.d.ts @@ -8,6 +8,9 @@ import { IScopedClusterClient } from 'kibana/server'; import type { MlClient } from '../../lib/ml_client'; import type { JobSavedObjectService } from '../../saved_objects'; +import { JobMessage } from '../../../common/types/audit_message'; + +export function isClearable(index?: string): boolean; export function jobAuditMessagesProvider( client: IScopedClusterClient, @@ -21,7 +24,10 @@ export function jobAuditMessagesProvider( start?: string; end?: string; } - ) => any; + ) => { messages: JobMessage[]; notificationIndices: string[] }; getAuditMessagesSummary: (jobIds?: string[]) => any; - clearJobAuditMessages: (jobId: string) => any; + clearJobAuditMessages: ( + jobId: string, + notificationIndices: string[] + ) => { success: boolean; last_cleared: number }; }; diff --git a/x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.js b/x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.js index 137df3a6f3151..311df2ac418c0 100644 --- a/x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.js +++ b/x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.js @@ -5,10 +5,7 @@ * 2.0. */ -import { - ML_NOTIFICATION_INDEX_PATTERN, - ML_NOTIFICATION_INDEX_02, -} from '../../../common/constants/index_patterns'; +import { ML_NOTIFICATION_INDEX_PATTERN } from '../../../common/constants/index_patterns'; import { MESSAGE_LEVEL } from '../../../common/constants/message_levels'; import moment from 'moment'; @@ -39,6 +36,14 @@ const anomalyDetectorTypeFilter = { }, }; +export function isClearable(index) { + if (typeof index === 'string') { + const match = index.match(/\d{6}$/); + return match !== null && match.length && Number(match[match.length - 1]) >= 2; + } + return false; +} + export function jobAuditMessagesProvider({ asInternalUser }, mlClient) { // search for audit messages, // jobId is optional. without it, all jobs will be listed. @@ -126,18 +131,25 @@ export function jobAuditMessagesProvider({ asInternalUser }, mlClient) { }); let messages = []; + const notificationIndices = []; + if (body.hits.total.value > 0) { - messages = body.hits.hits.map((hit) => ({ - clearable: hit._index === ML_NOTIFICATION_INDEX_02, - ...hit._source, - })); + let notificationIndex; + body.hits.hits.forEach((hit) => { + if (notificationIndex !== hit._index && isClearable(hit._index)) { + notificationIndices.push(hit._index); + notificationIndex = hit._index; + } + + messages.push(hit._source); + }); } messages = await jobSavedObjectService.filterJobsForSpace( 'anomaly-detector', messages, 'job_id' ); - return messages; + return { messages, notificationIndices }; } // search highest, most recent audit messages for all jobs for the last 24hrs. @@ -281,7 +293,7 @@ export function jobAuditMessagesProvider({ asInternalUser }, mlClient) { const clearedTime = new Date().getTime(); // Sets 'cleared' to true for messages in the last 24hrs and index new message for clear action - async function clearJobAuditMessages(jobId) { + async function clearJobAuditMessages(jobId, notificationIndices) { const newClearedMessage = { job_id: jobId, job_type: 'anomaly_detection', @@ -309,9 +321,9 @@ export function jobAuditMessagesProvider({ asInternalUser }, mlClient) { }, }; - await Promise.all([ + const promises = [ asInternalUser.updateByQuery({ - index: ML_NOTIFICATION_INDEX_02, + index: notificationIndices.join(','), ignore_unavailable: true, refresh: false, conflicts: 'proceed', @@ -323,12 +335,16 @@ export function jobAuditMessagesProvider({ asInternalUser }, mlClient) { }, }, }), - asInternalUser.index({ - index: ML_NOTIFICATION_INDEX_02, - body: newClearedMessage, - refresh: 'wait_for', - }), - ]); + ...notificationIndices.map((index) => + asInternalUser.index({ + index, + body: newClearedMessage, + refresh: 'wait_for', + }) + ), + ]; + + await Promise.all(promises); return { success: true, last_cleared: clearedTime }; } diff --git a/x-pack/plugins/ml/server/routes/job_audit_messages.ts b/x-pack/plugins/ml/server/routes/job_audit_messages.ts index 1548427797e16..4dcaca573fc17 100644 --- a/x-pack/plugins/ml/server/routes/job_audit_messages.ts +++ b/x-pack/plugins/ml/server/routes/job_audit_messages.ts @@ -121,8 +121,8 @@ export function jobAuditMessagesRoutes({ router, routeGuard }: RouteInitializati async ({ client, mlClient, request, response, jobSavedObjectService }) => { try { const { clearJobAuditMessages } = jobAuditMessagesProvider(client, mlClient); - const { jobId } = request.body; - const resp = await clearJobAuditMessages(jobId); + const { jobId, notificationIndices } = request.body; + const resp = await clearJobAuditMessages(jobId, notificationIndices); return response.ok({ body: resp, diff --git a/x-pack/plugins/ml/server/routes/schemas/job_audit_messages_schema.ts b/x-pack/plugins/ml/server/routes/schemas/job_audit_messages_schema.ts index 525ac73fde120..aeff76f057fc6 100644 --- a/x-pack/plugins/ml/server/routes/schemas/job_audit_messages_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/job_audit_messages_schema.ts @@ -20,4 +20,5 @@ export const jobAuditMessagesQuerySchema = schema.object({ export const clearJobAuditMessagesBodySchema = schema.object({ jobId: schema.string(), + notificationIndices: schema.arrayOf(schema.string()), }); From 9dcecaedc61a4962b1cc0ce84aa16694a2ace5d9 Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Fri, 30 Jul 2021 11:12:52 -0400 Subject: [PATCH 2/6] [ML] Data frame analytics: ensure results view loads correctly for job created in dev tools (#107024) * check analyzed_fields exists * update DataFrameAnalyticsConfig type to make analyzed_fields optional * updating accessing analyzed_fields for type update --- x-pack/plugins/ml/common/types/data_frame_analytics.ts | 6 +++--- .../exploration_page_wrapper/exploration_page_wrapper.tsx | 4 ++-- .../components/outlier_exploration/outlier_exploration.tsx | 4 ++-- .../ml/server/models/data_frame_analytics/validation.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/ml/common/types/data_frame_analytics.ts b/x-pack/plugins/ml/common/types/data_frame_analytics.ts index 3305eeaaf4794..ec2a244c75468 100644 --- a/x-pack/plugins/ml/common/types/data_frame_analytics.ts +++ b/x-pack/plugins/ml/common/types/data_frame_analytics.ts @@ -80,9 +80,9 @@ export interface DataFrameAnalyticsConfig { runtime_mappings?: RuntimeMappings; }; analysis: AnalysisConfig; - analyzed_fields: { - includes: string[]; - excludes: string[]; + analyzed_fields?: { + includes?: string[]; + excludes?: string[]; }; model_memory_limit: string; max_num_threads?: number; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx index 6c158f103aade..48477acfe7be8 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx @@ -116,8 +116,8 @@ export const ExplorationPageWrapper: FC = ({ const resultsField = jobConfig?.dest.results_field ?? ''; const scatterplotFieldOptions = useScatterplotFieldOptions( indexPattern, - jobConfig?.analyzed_fields.includes, - jobConfig?.analyzed_fields.excludes, + jobConfig?.analyzed_fields?.includes, + jobConfig?.analyzed_fields?.excludes, resultsField ); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx index 5f013c634e4c4..abd1870babfb9 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx @@ -92,8 +92,8 @@ export const OutlierExploration: FC = React.memo(({ jobId }) = const scatterplotFieldOptions = useScatterplotFieldOptions( indexPattern, - jobConfig?.analyzed_fields.includes, - jobConfig?.analyzed_fields.excludes, + jobConfig?.analyzed_fields?.includes, + jobConfig?.analyzed_fields?.excludes, resultsField ); diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/validation.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/validation.ts index 216a4379c7c89..b39debbe664d3 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/validation.ts +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/validation.ts @@ -448,7 +448,7 @@ export async function validateAnalyticsJob( ) { const messages = await getValidationCheckMessages( client.asCurrentUser, - job.analyzed_fields.includes, + job?.analyzed_fields?.includes || [], job.analysis, job.source ); From 5294f1ca177fe15bde40e643eb08d0e0730ad983 Mon Sep 17 00:00:00 2001 From: Mikhail Shustov Date: Fri, 30 Jul 2021 18:15:50 +0300 Subject: [PATCH 3/6] remove elasticsearch package imports from reporting (#107126) * remove elasticsearch package imports from reporting * remove dead code --- .../generate_csv/generate_csv.ts | 10 ++++----- .../reporting/server/routes/generation.ts | 21 ------------------- 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts index 254bc0ae21f6c..a38e0d58abf89 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts @@ -6,7 +6,7 @@ */ import { i18n } from '@kbn/i18n'; -import { SearchResponse } from 'elasticsearch'; +import type { estypes } from '@elastic/elasticsearch'; import { IScopedClusterClient, IUiSettingsClient } from 'src/core/server'; import { IScopedSearchClient } from 'src/plugins/data/server'; import { Datatable } from 'src/plugins/expressions/server'; @@ -93,7 +93,7 @@ export class CsvGenerator { }; const results = ( await this.clients.data.search(searchParams, { strategy: ES_SEARCH_STRATEGY }).toPromise() - ).rawResponse as SearchResponse; + ).rawResponse as estypes.SearchResponse; return results; } @@ -107,7 +107,7 @@ export class CsvGenerator { scroll_id: scrollId, }, }) - ).body as SearchResponse; + ).body; return results; } @@ -321,13 +321,13 @@ export class CsvGenerator { if (this.cancellationToken.isCancelled()) { break; } - let results: SearchResponse | undefined; + let results: estypes.SearchResponse | undefined; if (scrollId == null) { // open a scroll cursor in Elasticsearch results = await this.scan(index, searchSource, scrollSettings); scrollId = results?._scroll_id; if (results.hits?.total != null) { - totalRecords = results.hits.total; + totalRecords = results.hits.total as number; this.logger.debug(`Total search results: ${totalRecords}`); } } else { diff --git a/x-pack/plugins/reporting/server/routes/generation.ts b/x-pack/plugins/reporting/server/routes/generation.ts index 5c9fd25b76c39..ce6d1a2f2641f 100644 --- a/x-pack/plugins/reporting/server/routes/generation.ts +++ b/x-pack/plugins/reporting/server/routes/generation.ts @@ -6,7 +6,6 @@ */ import Boom from '@hapi/boom'; -import { errors as elasticsearchErrors } from 'elasticsearch'; import { kibanaResponseFactory } from 'src/core/server'; import { ReportingCore } from '../'; import { API_BASE_URL } from '../../common/constants'; @@ -16,8 +15,6 @@ import { registerGenerateFromJobParams } from './generate_from_jobparams'; import { registerGenerateCsvFromSavedObjectImmediate } from './csv_searchsource_immediate'; import { HandlerFunction } from './types'; -const esErrors = elasticsearchErrors as Record; - const getDownloadBaseUrl = (reporting: ReportingCore) => { const config = reporting.getConfig(); return config.kbnConfig.get('server', 'basePath') + `${API_BASE_URL}/jobs/download`; @@ -77,24 +74,6 @@ export function registerJobGenerationRoutes(reporting: ReportingCore, logger: Lo }); } - if (err instanceof esErrors['401']) { - return res.unauthorized({ - body: `Sorry, you aren't authenticated`, - }); - } - - if (err instanceof esErrors['403']) { - return res.forbidden({ - body: `Sorry, you are not authorized`, - }); - } - - if (err instanceof esErrors['404']) { - return res.notFound({ - body: err.message, - }); - } - // unknown error, can't convert to 4xx throw err; } From 03ca8c7d35be09ee042d27cfd95226981180c2a3 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Fri, 30 Jul 2021 08:17:29 -0700 Subject: [PATCH 4/6] [Reporting CSV] Fix scroll ID bug for deprecated csv export (#106892) * [Reporting CSV] Fix scroll ID bug for deprecated csv export * fix unit test Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../server/export_types/csv/execute_job.test.ts | 10 +++++----- .../export_types/csv/generate_csv/hit_iterator.ts | 10 ++++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/reporting/server/export_types/csv/execute_job.test.ts b/x-pack/plugins/reporting/server/export_types/csv/execute_job.test.ts index 32b5370371cce..0e8a7016b853b 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/execute_job.test.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/execute_job.test.ts @@ -175,7 +175,7 @@ describe('CSV Execute Job', function () { ); expect(mockEsClient.scroll).toHaveBeenCalledWith( - expect.objectContaining({ scroll_id: scrollId }) + expect.objectContaining({ body: { scroll_id: scrollId } }) ); }); @@ -261,7 +261,7 @@ describe('CSV Execute Job', function () { ); expect(mockEsClient.clearScroll).toHaveBeenCalledWith( - expect.objectContaining({ scroll_id: lastScrollId }) + expect.objectContaining({ body: { scroll_id: lastScrollId } }) ); }); @@ -295,7 +295,7 @@ describe('CSV Execute Job', function () { ); expect(mockEsClient.clearScroll).toHaveBeenCalledWith( - expect.objectContaining({ scroll_id: lastScrollId }) + expect.objectContaining({ body: { scroll_id: lastScrollId } }) ); }); }); @@ -753,7 +753,7 @@ describe('CSV Execute Job', function () { expect(mockEsClient.clearScroll).toHaveBeenCalledWith( expect.objectContaining({ - scroll_id: scrollId, + body: { scroll_id: scrollId }, }) ); }); @@ -1150,7 +1150,7 @@ describe('CSV Execute Job', function () { await runTask('job123', jobParams, cancellationToken); expect(mockEsClient.scroll).toHaveBeenCalledWith( - expect.objectContaining({ scroll: scrollDuration }) + expect.objectContaining({ body: { scroll: scrollDuration, scroll_id: 'scrollId' } }) ); }); }); diff --git a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/hit_iterator.ts b/x-pack/plugins/reporting/server/export_types/csv/generate_csv/hit_iterator.ts index 72935e64dd6b5..9014e4f85b3b2 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/hit_iterator.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/generate_csv/hit_iterator.ts @@ -60,12 +60,14 @@ export function createHitIterator(logger: LevelLogger) { ); } - async function scroll(scrollId: string | undefined) { + async function scroll(scrollId: string) { logger.debug('executing scroll request'); return parseResponse( await elasticsearchClient.scroll({ - scroll_id: scrollId, - scroll: scrollSettings.duration, + body: { + scroll_id: scrollId, + scroll: scrollSettings.duration, + }, }) ); } @@ -74,7 +76,7 @@ export function createHitIterator(logger: LevelLogger) { logger.debug('executing clearScroll request'); try { await elasticsearchClient.clearScroll({ - scroll_id: scrollId, + body: { scroll_id: scrollId }, }); } catch (err) { // Do not throw the error, as the job can still be completed successfully From 0f05ecf2c087699cde525672f47b514aa962a09c Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Fri, 30 Jul 2021 16:28:29 +0100 Subject: [PATCH 5/6] Security Telemetry: Trusted applications (#106526) * [7.15] Setup project for trusted apps telemetry. * Create dependency on exception list client + query trusted applications. * Add trusted telemetry apps task. Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../server/lib/telemetry/mocks.ts | 9 ++ .../server/lib/telemetry/sender.ts | 33 +++-- .../lib/telemetry/trusted_apps_task.test.ts | 109 ++++++++++++++++ .../server/lib/telemetry/trusted_apps_task.ts | 122 ++++++++++++++++++ .../security_solution/server/plugin.ts | 7 +- 5 files changed, 270 insertions(+), 10 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts index 6e98dcd59e3ec..642be5fc737f7 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts @@ -9,6 +9,7 @@ import { TelemetryEventsSender } from './sender'; import { TelemetryDiagTask } from './diagnostic_task'; import { TelemetryEndpointTask } from './endpoint_task'; +import { TelemetryTrustedAppsTask } from './trusted_apps_task'; import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy'; /** @@ -24,6 +25,7 @@ export const createMockTelemetryEventsSender = ( fetchDiagnosticAlerts: jest.fn(), fetchEndpointMetrics: jest.fn(), fetchEndpointPolicyResponses: jest.fn(), + fetchTrustedApplications: jest.fn(), queueTelemetryEvents: jest.fn(), processEvents: jest.fn(), isTelemetryOptedIn: jest.fn().mockReturnValue(enableTelemtry ?? jest.fn()), @@ -65,3 +67,10 @@ export class MockTelemetryDiagnosticTask extends TelemetryDiagTask { export class MockTelemetryEndpointTask extends TelemetryEndpointTask { public runTask = jest.fn(); } + +/** + * Creates a mocked Telemetry trusted app Task + */ +export class MockTelemetryTrustedAppTask extends TelemetryTrustedAppsTask { + public runTask = jest.fn(); +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index bdd301d9fea1d..e8ef18ec798ae 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -19,8 +19,11 @@ import { } from '../../../../task_manager/server'; import { TelemetryDiagTask } from './diagnostic_task'; import { TelemetryEndpointTask } from './endpoint_task'; +import { TelemetryTrustedAppsTask } from './trusted_apps_task'; import { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; import { AgentService, AgentPolicyServiceInterface } from '../../../../fleet/server'; +import { ExceptionListClient } from '../../../../lists/server'; +import { getTrustedAppsList } from '../../endpoint/routes/trusted_apps/service'; type BaseSearchTypes = string | number | boolean | object; export type SearchTypes = BaseSearchTypes | BaseSearchTypes[] | undefined; @@ -57,10 +60,12 @@ export class TelemetryEventsSender { private isOptedIn?: boolean = true; // Assume true until the first check private diagTask?: TelemetryDiagTask; private epMetricsTask?: TelemetryEndpointTask; + private trustedAppsTask?: TelemetryTrustedAppsTask; private agentService?: AgentService; private agentPolicyService?: AgentPolicyServiceInterface; private esClient?: ElasticsearchClient; - private savedObjectClient?: SavedObjectsClientContract; + private savedObjectsClient?: SavedObjectsClientContract; + private exceptionListClient?: ExceptionListClient; constructor(logger: Logger) { this.logger = logger.get('telemetry_events'); @@ -72,6 +77,7 @@ export class TelemetryEventsSender { if (taskManager) { this.diagTask = new TelemetryDiagTask(this.logger, taskManager, this); this.epMetricsTask = new TelemetryEndpointTask(this.logger, taskManager, this); + this.trustedAppsTask = new TelemetryTrustedAppsTask(this.logger, taskManager, this); } } @@ -79,18 +85,21 @@ export class TelemetryEventsSender { core?: CoreStart, telemetryStart?: TelemetryPluginStart, taskManager?: TaskManagerStartContract, - endpointContextService?: EndpointAppContextService + endpointContextService?: EndpointAppContextService, + exceptionListClient?: ExceptionListClient ) { this.telemetryStart = telemetryStart; this.esClient = core?.elasticsearch.client.asInternalUser; this.agentService = endpointContextService?.getAgentService(); this.agentPolicyService = endpointContextService?.getAgentPolicyService(); - this.savedObjectClient = (core?.savedObjects.createInternalRepository() as unknown) as SavedObjectsClientContract; + this.savedObjectsClient = (core?.savedObjects.createInternalRepository() as unknown) as SavedObjectsClientContract; + this.exceptionListClient = exceptionListClient; if (taskManager && this.diagTask && this.epMetricsTask) { this.logger.debug(`Starting diagnostic and endpoint telemetry tasks`); this.diagTask.start(taskManager); this.epMetricsTask.start(taskManager); + this.trustedAppsTask?.start(taskManager); } this.logger.debug(`Starting local task`); @@ -139,7 +148,7 @@ export class TelemetryEventsSender { } public async fetchEndpointMetrics(executeFrom: string, executeTo: string) { - if (this.esClient === undefined) { + if (this.esClient === undefined || this.esClient === null) { throw Error('could not fetch policy responses. es client is not available'); } @@ -186,7 +195,7 @@ export class TelemetryEventsSender { } public async fetchFleetAgents() { - if (this.esClient === undefined) { + if (this.esClient === undefined || this.esClient === null) { throw Error('could not fetch policy responses. es client is not available'); } @@ -199,15 +208,15 @@ export class TelemetryEventsSender { } public async fetchPolicyConfigs(id: string) { - if (this.savedObjectClient === undefined) { + if (this.savedObjectsClient === undefined || this.savedObjectsClient === null) { throw Error('could not fetch endpoint policy configs. saved object client is not available'); } - return this.agentPolicyService?.get(this.savedObjectClient, id); + return this.agentPolicyService?.get(this.savedObjectsClient, id); } public async fetchEndpointPolicyResponses(executeFrom: string, executeTo: string) { - if (this.esClient === undefined) { + if (this.esClient === undefined || this.esClient === null) { throw Error('could not fetch policy responses. es client is not available'); } @@ -253,6 +262,14 @@ export class TelemetryEventsSender { return this.esClient.search(query); } + public async fetchTrustedApplications() { + if (this?.exceptionListClient === undefined || this?.exceptionListClient === null) { + throw Error('could not fetch trusted applications. exception list client not available.'); + } + + return getTrustedAppsList(this.exceptionListClient, { page: 1, per_page: 10_000 }); + } + public queueTelemetryEvents(events: TelemetryEvent[]) { const qlength = this.queue.length; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.test.ts new file mode 100644 index 0000000000000..5cd67a9c9c215 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from 'src/core/server/mocks'; +import { TaskStatus } from '../../../../task_manager/server'; +import { taskManagerMock } from '../../../../task_manager/server/mocks'; + +import { TelemetryTrustedAppsTask, TelemetryTrustedAppsTaskConstants } from './trusted_apps_task'; +import { createMockTelemetryEventsSender, MockTelemetryTrustedAppTask } from './mocks'; + +describe('test trusted apps telemetry task functionality', () => { + let logger: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + }); + + test('the trusted apps task can register', () => { + const telemetryTrustedAppsTask = new TelemetryTrustedAppsTask( + logger, + taskManagerMock.createSetup(), + createMockTelemetryEventsSender(true) + ); + + expect(telemetryTrustedAppsTask).toBeInstanceOf(TelemetryTrustedAppsTask); + }); + + test('the trusted apps task should be registered', () => { + const mockTaskManager = taskManagerMock.createSetup(); + new TelemetryTrustedAppsTask(logger, mockTaskManager, createMockTelemetryEventsSender(true)); + + expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled(); + }); + + test('the trusted apps task should be scheduled', async () => { + const mockTaskManagerSetup = taskManagerMock.createSetup(); + const telemetryTrustedAppsTask = new TelemetryTrustedAppsTask( + logger, + mockTaskManagerSetup, + createMockTelemetryEventsSender(true) + ); + + const mockTaskManagerStart = taskManagerMock.createStart(); + await telemetryTrustedAppsTask.start(mockTaskManagerStart); + expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled(); + }); + + test('the trusted apps task should not query elastic if telemetry is not opted in', async () => { + const mockSender = createMockTelemetryEventsSender(false); + const mockTaskManager = taskManagerMock.createSetup(); + new MockTelemetryTrustedAppTask(logger, mockTaskManager, mockSender); + + const mockTaskInstance = { + id: TelemetryTrustedAppsTaskConstants.TYPE, + runAt: new Date(), + attempts: 0, + ownerId: '', + status: TaskStatus.Running, + startedAt: new Date(), + scheduledAt: new Date(), + retryAt: new Date(), + params: {}, + state: {}, + taskType: TelemetryTrustedAppsTaskConstants.TYPE, + }; + const createTaskRunner = + mockTaskManager.registerTaskDefinitions.mock.calls[0][0][ + TelemetryTrustedAppsTaskConstants.TYPE + ].createTaskRunner; + const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); + await taskRunner.run(); + expect(mockSender.fetchTrustedApplications).not.toHaveBeenCalled(); + }); + + test('the trusted apps task should query elastic if telemetry opted in', async () => { + const mockSender = createMockTelemetryEventsSender(true); + const mockTaskManager = taskManagerMock.createSetup(); + const telemetryTrustedAppsTask = new MockTelemetryTrustedAppTask( + logger, + mockTaskManager, + mockSender + ); + + const mockTaskInstance = { + id: TelemetryTrustedAppsTaskConstants.TYPE, + runAt: new Date(), + attempts: 0, + ownerId: '', + status: TaskStatus.Running, + startedAt: new Date(), + scheduledAt: new Date(), + retryAt: new Date(), + params: {}, + state: {}, + taskType: TelemetryTrustedAppsTaskConstants.TYPE, + }; + const createTaskRunner = + mockTaskManager.registerTaskDefinitions.mock.calls[0][0][ + TelemetryTrustedAppsTaskConstants.TYPE + ].createTaskRunner; + const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); + await taskRunner.run(); + expect(telemetryTrustedAppsTask.runTask).toHaveBeenCalled(); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.ts new file mode 100644 index 0000000000000..f91f3e8428d04 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/trusted_apps_task.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import { Logger } from 'src/core/server'; +import { + ConcreteTaskInstance, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../../task_manager/server'; + +import { getPreviousEpMetaTaskTimestamp, batchTelemetryRecords } from './helpers'; +import { TelemetryEventsSender } from './sender'; + +export const TelemetryTrustedAppsTaskConstants = { + TIMEOUT: '1m', + TYPE: 'security:trusted-apps-telemetry', + INTERVAL: '24h', + VERSION: '1.0.0', +}; + +/** Telemetry Trusted Apps Task + * + * The Trusted Apps task is a daily batch job that collects and transmits non-sensitive + * trusted apps hashes + file paths for supported operating systems. This helps test + * efficacy of our protections. + */ +export class TelemetryTrustedAppsTask { + private readonly logger: Logger; + private readonly sender: TelemetryEventsSender; + + constructor( + logger: Logger, + taskManager: TaskManagerSetupContract, + sender: TelemetryEventsSender + ) { + this.logger = logger; + this.sender = sender; + + taskManager.registerTaskDefinitions({ + [TelemetryTrustedAppsTaskConstants.TYPE]: { + title: 'Security Solution Telemetry Endpoint Metrics and Info task', + timeout: TelemetryTrustedAppsTaskConstants.TIMEOUT, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + const { state } = taskInstance; + + return { + run: async () => { + const taskExecutionTime = moment().utc().toISOString(); + const lastExecutionTimestamp = getPreviousEpMetaTaskTimestamp( + taskExecutionTime, + taskInstance.state?.lastExecutionTimestamp + ); + + const hits = await this.runTask( + taskInstance.id, + lastExecutionTimestamp, + taskExecutionTime + ); + + return { + state: { + lastExecutionTimestamp: taskExecutionTime, + runs: (state.runs || 0) + 1, + hits, + }, + }; + }, + cancel: async () => {}, + }; + }, + }, + }); + } + + public start = async (taskManager: TaskManagerStartContract) => { + try { + await taskManager.ensureScheduled({ + id: this.getTaskId(), + taskType: TelemetryTrustedAppsTaskConstants.TYPE, + scope: ['securitySolution'], + schedule: { + interval: TelemetryTrustedAppsTaskConstants.INTERVAL, + }, + state: { runs: 0 }, + params: { version: TelemetryTrustedAppsTaskConstants.VERSION }, + }); + } catch (e) { + this.logger.error(`Error scheduling task, received ${e.message}`); + } + }; + + private getTaskId = (): string => { + return `${TelemetryTrustedAppsTaskConstants.TYPE}:${TelemetryTrustedAppsTaskConstants.VERSION}`; + }; + + public runTask = async (taskId: string, executeFrom: string, executeTo: string) => { + if (taskId !== this.getTaskId()) { + this.logger.debug(`Outdated task running: ${taskId}`); + return 0; + } + + const isOptedIn = await this.sender.isTelemetryOptedIn(); + if (!isOptedIn) { + this.logger.debug(`Telemetry is not opted-in.`); + return 0; + } + + const response = await this.sender.fetchTrustedApplications(); + this.logger.debug(`Trusted Apps: ${response}`); + + batchTelemetryRecords(response.data, 1_000).forEach((telemetryBatch) => + this.sender.sendOnDemand('lists-trustedapps', telemetryBatch) + ); + + return response.data.length; + }; +} diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 2c0be2ac93321..a68280379fad3 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -391,6 +391,8 @@ export class Plugin implements IPlugin Date: Fri, 30 Jul 2021 12:02:51 -0400 Subject: [PATCH 6/6] [Fleet] Missing SO Installation migration (#107214) --- .../plugins/fleet/server/saved_objects/index.ts | 5 ++++- .../saved_objects/migrations/to_v7_10_0.ts | 10 ---------- .../saved_objects/migrations/to_v7_14_0.ts | 16 +++++++++++++++- .../apis/epm/install_by_upload.ts | 1 + 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 9f9f0dab6efac..54cb0846207a3 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -42,7 +42,7 @@ import { migrateSettingsToV7130, migrateOutputToV7130, } from './migrations/to_v7_13_0'; -import { migratePackagePolicyToV7140 } from './migrations/to_v7_14_0'; +import { migratePackagePolicyToV7140, migrateInstallationToV7140 } from './migrations/to_v7_14_0'; import { migratePackagePolicyToV7150 } from './migrations/to_v7_15_0'; /* @@ -320,6 +320,9 @@ const getSavedObjectTypes = ( install_source: { type: 'keyword' }, }, }, + migrations: { + '7.14.0': migrateInstallationToV7140, + }, }, [ASSETS_SAVED_OBJECT_TYPE]: { name: ASSETS_SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts index 39e65efcf2ab1..64338690977c9 100644 --- a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts @@ -15,7 +15,6 @@ import type { EnrollmentAPIKey, Settings, AgentAction, - Installation, } from '../../types'; export const migrateAgentToV7100: SavedObjectMigrationFn< @@ -127,12 +126,3 @@ export const migrateAgentActionToV7100 = ( }, }); }; - -export const migrateInstallationToV7100: SavedObjectMigrationFn< - Exclude, - Installation -> = (installationDoc) => { - installationDoc.attributes.install_source = 'registry'; - - return installationDoc; -}; diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_14_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_14_0.ts index 3255e15c6ceec..90c9ac5f8e89b 100644 --- a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_14_0.ts +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_14_0.ts @@ -7,7 +7,7 @@ import type { SavedObjectMigrationFn } from 'kibana/server'; -import type { PackagePolicy } from '../../../common'; +import type { PackagePolicy, Installation } from '../../../common'; import { migrateEndpointPackagePolicyToV7140 } from './security_solution'; @@ -27,3 +27,17 @@ export const migratePackagePolicyToV7140: SavedObjectMigrationFn = ( + doc +) => { + // Fix a missing migration for user that used Fleet before 7.9 + if (!doc.attributes.install_source) { + doc.attributes.install_source = 'registry'; + } + if (!doc.attributes.install_version) { + doc.attributes.install_version = doc.attributes.version; + } + + return doc; +}; diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts index 182838f21dbda..06130775ec3cb 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts @@ -93,6 +93,7 @@ export default function (providerContext: FtrProviderContext) { delete packageInfoRes.body.response.savedObject.version; delete packageInfoRes.body.response.savedObject.updated_at; delete packageInfoRes.body.response.savedObject.coreMigrationVersion; + delete packageInfoRes.body.response.savedObject.migrationVersion; expectSnapshot(packageInfoRes.body.response).toMatch(); });