Merge branch 'elastic:master' into so-errors/improve-response-to-not-…

…found

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -42,7 +42,7 @@ import {
   migrateSettingsToV7130,
   migrateOutputToV7130,
 } from './migrations/to_v7_13_0';
-import { migratePackagePolicyToV7140 } from './migrations/to_v7_14_0';
+import { migratePackagePolicyToV7140, migrateInstallationToV7140 } from './migrations/to_v7_14_0';
 import { migratePackagePolicyToV7150 } from './migrations/to_v7_15_0';
 
 /*
@@ -320,6 +320,9 @@ const getSavedObjectTypes = (
         install_source: { type: 'keyword' },
       },
     },
+    migrations: {
+      '7.14.0': migrateInstallationToV7140,
+    },
   },
   [ASSETS_SAVED_OBJECT_TYPE]: {
     name: ASSETS_SAVED_OBJECT_TYPE,

x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts

@@ -15,7 +15,6 @@ import type {
   EnrollmentAPIKey,
   Settings,
   AgentAction,
-  Installation,
 } from '../../types';
 
 export const migrateAgentToV7100: SavedObjectMigrationFn<
@@ -127,12 +126,3 @@ export const migrateAgentActionToV7100 = (
     },
   });
 };
-
-export const migrateInstallationToV7100: SavedObjectMigrationFn<
-  Exclude<Installation, 'install_source'>,
-  Installation
-> = (installationDoc) => {
-  installationDoc.attributes.install_source = 'registry';
-
-  return installationDoc;
-};

x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_14_0.ts

@@ -7,7 +7,7 @@
 
 import type { SavedObjectMigrationFn } from 'kibana/server';
 
-import type { PackagePolicy } from '../../../common';
+import type { PackagePolicy, Installation } from '../../../common';
 
 import { migrateEndpointPackagePolicyToV7140 } from './security_solution';
 
@@ -27,3 +27,17 @@ export const migratePackagePolicyToV7140: SavedObjectMigrationFn<PackagePolicy,
 
   return updatedPackagePolicyDoc;
 };
+
+export const migrateInstallationToV7140: SavedObjectMigrationFn<Installation, Installation> = (
+  doc
+) => {
+  // Fix a missing migration for user that used Fleet before 7.9
+  if (!doc.attributes.install_source) {
+    doc.attributes.install_source = 'registry';
+  }
+  if (!doc.attributes.install_version) {
+    doc.attributes.install_version = doc.attributes.version;
+  }
+
+  return doc;
+};

x-pack/plugins/ml/common/constants/index_patterns.ts

@@ -11,4 +11,3 @@ export const ML_ANNOTATIONS_INDEX_PATTERN = '.ml-annotations-6';
 
 export const ML_RESULTS_INDEX_PATTERN = '.ml-anomalies-*';
 export const ML_NOTIFICATION_INDEX_PATTERN = '.ml-notifications*';
-export const ML_NOTIFICATION_INDEX_02 = '.ml-notifications-000002';

x-pack/plugins/ml/common/types/data_frame_analytics.ts

@@ -80,9 +80,9 @@ export interface DataFrameAnalyticsConfig {
     runtime_mappings?: RuntimeMappings;
   };
   analysis: AnalysisConfig;
-  analyzed_fields: {
-    includes: string[];
-    excludes: string[];
+  analyzed_fields?: {
+    includes?: string[];
+    excludes?: string[];
   };
   model_memory_limit: string;
   max_num_threads?: number;

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx

@@ -116,8 +116,8 @@ export const ExplorationPageWrapper: FC<Props> = ({
   const resultsField = jobConfig?.dest.results_field ?? '';
   const scatterplotFieldOptions = useScatterplotFieldOptions(
     indexPattern,
-    jobConfig?.analyzed_fields.includes,
-    jobConfig?.analyzed_fields.excludes,
+    jobConfig?.analyzed_fields?.includes,
+    jobConfig?.analyzed_fields?.excludes,
     resultsField
   );
 

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx

@@ -92,8 +92,8 @@ export const OutlierExploration: FC<ExplorationProps> = React.memo(({ jobId }) =
 
   const scatterplotFieldOptions = useScatterplotFieldOptions(
     indexPattern,
-    jobConfig?.analyzed_fields.includes,
-    jobConfig?.analyzed_fields.excludes,
+    jobConfig?.analyzed_fields?.includes,
+    jobConfig?.analyzed_fields?.excludes,
     resultsField
   );
 

x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_messages_pane.tsx

@@ -30,6 +30,7 @@ export const JobMessagesPane: FC<JobMessagesPaneProps> = React.memo(
     const canCreateJob = checkPermission('canCreateJob');
 
     const [messages, setMessages] = useState<JobMessage[]>([]);
+    const [notificationIndices, setNotificationIndices] = useState<string[]>([]);
     const [isLoading, setIsLoading] = useState(false);
     const [errorMessage, setErrorMessage] = useState('');
     const [isClearing, setIsClearing] = useState<boolean>(false);
@@ -42,7 +43,10 @@ export const JobMessagesPane: FC<JobMessagesPaneProps> = React.memo(
     const fetchMessages = async () => {
       setIsLoading(true);
       try {
-        setMessages(await ml.jobs.jobAuditMessages({ jobId, start, end }));
+        const messagesResp = await ml.jobs.jobAuditMessages({ jobId, start, end });
+
+        setMessages(messagesResp.messages);
+        setNotificationIndices(messagesResp.notificationIndices);
         setIsLoading(false);
       } catch (error) {
         setIsLoading(false);
@@ -63,7 +67,7 @@ export const JobMessagesPane: FC<JobMessagesPaneProps> = React.memo(
     const clearMessages = useCallback(async () => {
       setIsClearing(true);
       try {
-        await clearJobAuditMessages(jobId);
+        await clearJobAuditMessages(jobId, notificationIndices);
         setIsClearing(false);
         if (typeof refreshJobList === 'function') {
           refreshJobList();
@@ -77,13 +81,13 @@ export const JobMessagesPane: FC<JobMessagesPaneProps> = React.memo(
           })
         );
       }
-    }, [jobId]);
+    }, [jobId, JSON.stringify(notificationIndices)]);
 
     useEffect(() => {
       fetchMessages();
     }, []);
 
-    const disabled = messages.length > 0 && messages[0].clearable === false;
+    const disabled = notificationIndices.length === 0;
 
     const clearButton = (
       <EuiButton

x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts

@@ -153,15 +153,15 @@ export const jobsApiProvider = (httpService: HttpService) => ({
       ...(start !== undefined && end !== undefined ? { start, end } : {}),
     };
 
-    return httpService.http<JobMessage[]>({
+    return httpService.http<{ messages: JobMessage[]; notificationIndices: string[] }>({
       path: `${ML_BASE_PATH}/job_audit_messages/messages${jobIdString}`,
       method: 'GET',
       query,
     });
   },
 
-  clearJobAuditMessages(jobId: string) {
-    const body = JSON.stringify({ jobId });
+  clearJobAuditMessages(jobId: string, notificationIndices: string[]) {
+    const body = JSON.stringify({ jobId, notificationIndices });
     return httpService.http<{ success: boolean; latest_cleared: number }>({
       path: `${ML_BASE_PATH}/job_audit_messages/clear_messages`,
       method: 'PUT',

x-pack/plugins/ml/server/models/data_frame_analytics/validation.ts

@@ -448,7 +448,7 @@ export async function validateAnalyticsJob(
 ) {
   const messages = await getValidationCheckMessages(
     client.asCurrentUser,
-    job.analyzed_fields.includes,
+    job?.analyzed_fields?.includes || [],
     job.analysis,
     job.source
   );

x-pack/plugins/ml/server/models/job_audit_messages/is_clearable.test.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isClearable } from './job_audit_messages';
+
+const supportedNotificationIndices = [
+  '.ml-notifications-000002',
+  '.ml-notifications-000003',
+  '.ml-notifications-000004',
+];
+
+const unsupportedIndices = ['.ml-notifications-000001', 'index-does-not-exist'];
+
+describe('jobAuditMessages - isClearable', () => {
+  it('should return true for indices ending in a six digit number with the last number >= 2', () => {
+    supportedNotificationIndices.forEach((index) => {
+      expect(isClearable(index)).toEqual(true);
+    });
+  });
+
+  it('should return false for indices not ending in a six digit number with the last number >= 2', () => {
+    unsupportedIndices.forEach((index) => {
+      expect(isClearable(index)).toEqual(false);
+    });
+  });
+
+  it('should return false for empty string or missing argument', () => {
+    expect(isClearable('')).toEqual(false);
+    expect(isClearable()).toEqual(false);
+  });
+});

x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.d.ts

@@ -8,6 +8,9 @@
 import { IScopedClusterClient } from 'kibana/server';
 import type { MlClient } from '../../lib/ml_client';
 import type { JobSavedObjectService } from '../../saved_objects';
+import { JobMessage } from '../../../common/types/audit_message';
+
+export function isClearable(index?: string): boolean;
 
 export function jobAuditMessagesProvider(
   client: IScopedClusterClient,
@@ -21,7 +24,10 @@ export function jobAuditMessagesProvider(
       start?: string;
       end?: string;
     }
-  ) => any;
+  ) => { messages: JobMessage[]; notificationIndices: string[] };
   getAuditMessagesSummary: (jobIds?: string[]) => any;
-  clearJobAuditMessages: (jobId: string) => any;
+  clearJobAuditMessages: (
+    jobId: string,
+    notificationIndices: string[]
+  ) => { success: boolean; last_cleared: number };
 };

x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.js

@@ -5,10 +5,7 @@
  * 2.0.
  */
 
-import {
-  ML_NOTIFICATION_INDEX_PATTERN,
-  ML_NOTIFICATION_INDEX_02,
-} from '../../../common/constants/index_patterns';
+import { ML_NOTIFICATION_INDEX_PATTERN } from '../../../common/constants/index_patterns';
 import { MESSAGE_LEVEL } from '../../../common/constants/message_levels';
 import moment from 'moment';
 
@@ -39,6 +36,14 @@ const anomalyDetectorTypeFilter = {
   },
 };
 
+export function isClearable(index) {
+  if (typeof index === 'string') {
+    const match = index.match(/\d{6}$/);
+    return match !== null && match.length && Number(match[match.length - 1]) >= 2;
+  }
+  return false;
+}
+
 export function jobAuditMessagesProvider({ asInternalUser }, mlClient) {
   // search for audit messages,
   // jobId is optional. without it, all jobs will be listed.
@@ -126,18 +131,25 @@ export function jobAuditMessagesProvider({ asInternalUser }, mlClient) {
     });
 
     let messages = [];
+    const notificationIndices = [];
+
     if (body.hits.total.value > 0) {
-      messages = body.hits.hits.map((hit) => ({
-        clearable: hit._index === ML_NOTIFICATION_INDEX_02,
-        ...hit._source,
-      }));
+      let notificationIndex;
+      body.hits.hits.forEach((hit) => {
+        if (notificationIndex !== hit._index && isClearable(hit._index)) {
+          notificationIndices.push(hit._index);
+          notificationIndex = hit._index;
+        }
+
+        messages.push(hit._source);
+      });
     }
     messages = await jobSavedObjectService.filterJobsForSpace(
       'anomaly-detector',
       messages,
       'job_id'
     );
-    return messages;
+    return { messages, notificationIndices };
   }
 
   // search highest, most recent audit messages for all jobs for the last 24hrs.
@@ -281,7 +293,7 @@ export function jobAuditMessagesProvider({ asInternalUser }, mlClient) {
   const clearedTime = new Date().getTime();
 
   // Sets 'cleared' to true for messages in the last 24hrs and index new message for clear action
-  async function clearJobAuditMessages(jobId) {
+  async function clearJobAuditMessages(jobId, notificationIndices) {
     const newClearedMessage = {
       job_id: jobId,
       job_type: 'anomaly_detection',
@@ -309,9 +321,9 @@ export function jobAuditMessagesProvider({ asInternalUser }, mlClient) {
       },
     };
 
-    await Promise.all([
+    const promises = [
       asInternalUser.updateByQuery({
-        index: ML_NOTIFICATION_INDEX_02,
+        index: notificationIndices.join(','),
         ignore_unavailable: true,
         refresh: false,
         conflicts: 'proceed',
@@ -323,12 +335,16 @@ export function jobAuditMessagesProvider({ asInternalUser }, mlClient) {
           },
         },
       }),
-      asInternalUser.index({
-        index: ML_NOTIFICATION_INDEX_02,
-        body: newClearedMessage,
-        refresh: 'wait_for',
-      }),
-    ]);
+      ...notificationIndices.map((index) =>
+        asInternalUser.index({
+          index,
+          body: newClearedMessage,
+          refresh: 'wait_for',
+        })
+      ),
+    ];
+
+    await Promise.all(promises);
 
     return { success: true, last_cleared: clearedTime };
   }

x-pack/plugins/ml/server/routes/job_audit_messages.ts

@@ -121,8 +121,8 @@ export function jobAuditMessagesRoutes({ router, routeGuard }: RouteInitializati
       async ({ client, mlClient, request, response, jobSavedObjectService }) => {
         try {
           const { clearJobAuditMessages } = jobAuditMessagesProvider(client, mlClient);
-          const { jobId } = request.body;
-          const resp = await clearJobAuditMessages(jobId);
+          const { jobId, notificationIndices } = request.body;
+          const resp = await clearJobAuditMessages(jobId, notificationIndices);
 
           return response.ok({
             body: resp,

x-pack/plugins/ml/server/routes/schemas/job_audit_messages_schema.ts

@@ -20,4 +20,5 @@ export const jobAuditMessagesQuerySchema = schema.object({
 
 export const clearJobAuditMessagesBodySchema = schema.object({
   jobId: schema.string(),
+  notificationIndices: schema.arrayOf(schema.string()),
 });