Merge branch 'master' into pjhampton/bug-policy-responses

docs/migration/migrate_8_0.asciidoc

@@ -344,4 +344,24 @@ The output directory after extracting an archive no longer includes the target p
 *Impact:*
 Configuration management tools and automation will need to be updated to use the new directory.
 
+[float]
+=== `elasticsearch.preserveHost` is no longer valid
+*Details:* The deprecated `elasticsearch.preserveHost` setting in the `kibana.yml` file has been removed.
+
+*Impact:* Configure {kibana-ref}/settings.html#elasticsearch-requestHeadersWhitelist[`elasticsearch.requestHeadersWhitelist`] to whitelist client-side headers.
+
+[float]
+=== `elasticsearch.startupTimeout` is no longer valid
+*Details:* The deprecated `elasticsearch.startupTimeout` setting in the `kibana.yml` file has been removed.
+
+*Impact:* Kibana will keep on trying to connect to Elasticsearch until it manages to connect.
+
+[float]
+=== `savedObjects.indexCheckTimeout` is no longer valid
+*Details:* The deprecated `savedObjects.indexCheckTimeout` setting in the `kibana.yml` file has been removed.
+
+[float]
+=== `server.xsrf.token` is no longer valid
+*Details:* The deprecated `server.xsrf.token` setting in the `kibana.yml` file has been removed.
+
 // end::notable-breaking-changes[]

package.json

@@ -96,9 +96,9 @@
     "@elastic/apm-rum-react": "^1.3.1",
     "@elastic/charts": "34.2.1",
     "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath",
-    "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.20",
+    "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.21",
     "@elastic/ems-client": "7.15.0",
-    "@elastic/eui": "37.6.0",
+    "@elastic/eui": "38.0.1",
     "@elastic/filesaver": "1.1.2",
     "@elastic/good": "^9.0.1-kibana3",
     "@elastic/maki": "6.3.0",
@@ -266,6 +266,7 @@
     "js-levenshtein": "^1.1.6",
     "js-search": "^1.4.3",
     "js-sha256": "^0.9.0",
+    "js-sql-parser": "^1.4.1",
     "js-yaml": "^3.14.0",
     "json-stable-stringify": "^1.0.1",
     "json-stringify-pretty-compact": "1.2.0",
@@ -298,7 +299,6 @@
     "nock": "12.0.3",
     "node-fetch": "^2.6.1",
     "node-forge": "^0.10.0",
-    "node-sql-parser": "^3.6.1",
     "nodemailer": "^6.6.2",
     "normalize-path": "^3.0.0",
     "object-hash": "^1.3.1",

src/core/public/chrome/ui/header/header.test.tsx

@@ -15,10 +15,6 @@ import { applicationServiceMock } from '../../../mocks';
 import { Header } from './header';
 import { ChromeBreadcrumbsAppendExtension } from '../../types';
 
-jest.mock('@elastic/eui/lib/services/accessibility/html_id_generator', () => ({
-  htmlIdGenerator: () => () => 'mockId',
-}));
-
 function mockProps() {
   const http = httpServiceMock.createSetupContract({ basePath: '/test' });
   const application = applicationServiceMock.createInternalStartContract();

src/core/public/saved_objects/saved_objects_client.test.ts

@@ -148,13 +148,15 @@ describe('SavedObjectsClient', () => {
   });
 
   describe('#resolve', () => {
-    beforeEach(() => {
+    function mockResolvedObjects(...objects: Array<Record<string, unknown>>) {
       http.fetch.mockResolvedValue({
-        resolved_objects: [
-          { saved_object: doc, outcome: 'conflict', alias_target_id: 'another-id' },
-        ],
+        resolved_objects: objects.map((obj) => ({
+          saved_object: obj,
+          outcome: 'conflict',
+          alias_target_id: 'another-id',
+        })),
       });
-    });
+    }
 
     test('rejects if `type` parameter is undefined', () => {
       return expect(
@@ -176,6 +178,7 @@ describe('SavedObjectsClient', () => {
     });
 
     test('makes HTTP call', async () => {
+      mockResolvedObjects(doc);
       await savedObjectsClient.resolve(doc.type, doc.id);
       expect(http.fetch.mock.calls[0]).toMatchInlineSnapshot(`
         Array [
@@ -191,10 +194,12 @@ describe('SavedObjectsClient', () => {
 
     test('batches several #resolve calls into a single HTTP call', async () => {
       // Await #resolve call to ensure batchQueue is empty and throttle has reset
+      mockResolvedObjects({ ...doc, type: 'type2' });
       await savedObjectsClient.resolve('type2', doc.id);
       http.fetch.mockClear();
 
       // Make two #resolve calls right after one another
+      mockResolvedObjects({ ...doc, type: 'type1' }, { ...doc, type: 'type0' });
       savedObjectsClient.resolve('type1', doc.id);
       await savedObjectsClient.resolve('type0', doc.id);
       expect(http.fetch.mock.calls).toMatchInlineSnapshot(`
@@ -213,9 +218,11 @@ describe('SavedObjectsClient', () => {
 
     test('removes duplicates when calling `_bulk_resolve`', async () => {
       // Await #resolve call to ensure batchQueue is empty and throttle has reset
+      mockResolvedObjects({ ...doc, type: 'type2' });
       await savedObjectsClient.resolve('type2', doc.id);
       http.fetch.mockClear();
 
+      mockResolvedObjects(doc, { ...doc, type: 'some-type', id: 'some-id' }); // the client will only request two objects, so we only mock two results
       savedObjectsClient.resolve(doc.type, doc.id);
       savedObjectsClient.resolve('some-type', 'some-id');
       await savedObjectsClient.resolve(doc.type, doc.id);
@@ -235,9 +242,11 @@ describe('SavedObjectsClient', () => {
 
     test('resolves with correct object when there are duplicates present', async () => {
       // Await #resolve call to ensure batchQueue is empty and throttle has reset
+      mockResolvedObjects({ ...doc, type: 'type2' });
       await savedObjectsClient.resolve('type2', doc.id);
       http.fetch.mockClear();
 
+      mockResolvedObjects(doc);
       const call1 = savedObjectsClient.resolve(doc.type, doc.id);
       const objFromCall2 = await savedObjectsClient.resolve(doc.type, doc.id);
       const objFromCall1 = await call1;
@@ -252,8 +261,10 @@ describe('SavedObjectsClient', () => {
     test('do not share instances or references between duplicate callers', async () => {
       // Await #resolve call to ensure batchQueue is empty and throttle has reset
       await savedObjectsClient.resolve('type2', doc.id);
+      mockResolvedObjects({ ...doc, type: 'type2' });
       http.fetch.mockClear();
 
+      mockResolvedObjects(doc);
       const call1 = savedObjectsClient.resolve(doc.type, doc.id);
       const objFromCall2 = await savedObjectsClient.resolve(doc.type, doc.id);
       const objFromCall1 = await call1;
@@ -263,6 +274,7 @@ describe('SavedObjectsClient', () => {
     });
 
     test('resolves with ResolvedSimpleSavedObject instance', async () => {
+      mockResolvedObjects(doc);
       const result = await savedObjectsClient.resolve(doc.type, doc.id);
       expect(result.saved_object).toBeInstanceOf(SimpleSavedObject);
       expect(result.saved_object.type).toBe(doc.type);

src/core/public/saved_objects/saved_objects_client.ts

@@ -151,9 +151,7 @@ interface ObjectTypeAndId {
   type: string;
 }
 
-const getObjectsToFetch = (
-  queue: Array<BatchGetQueueEntry | BatchResolveQueueEntry>
-): ObjectTypeAndId[] => {
+const getObjectsToFetch = (queue: BatchGetQueueEntry[]): ObjectTypeAndId[] => {
   const objects: ObjectTypeAndId[] = [];
   const inserted = new Set<string>();
   queue.forEach(({ id, type }) => {
@@ -165,6 +163,24 @@ const getObjectsToFetch = (
   return objects;
 };
 
+const getObjectsToResolve = (queue: BatchResolveQueueEntry[]) => {
+  const responseIndices: number[] = [];
+  const objectsToResolve: ObjectTypeAndId[] = [];
+  const inserted = new Map<string, number>();
+  queue.forEach(({ id, type }, currentIndex) => {
+    const key = `${type}|${id}`;
+    const indexForTypeAndId = inserted.get(key);
+    if (indexForTypeAndId === undefined) {
+      inserted.set(key, currentIndex);
+      objectsToResolve.push({ id, type });
+      responseIndices.push(currentIndex);
+    } else {
+      responseIndices.push(indexForTypeAndId);
+    }
+  });
+  return { objectsToResolve, responseIndices };
+};
+
 /**
  * Saved Objects is Kibana's data persisentence mechanism allowing plugins to
  * use Elasticsearch for storing plugin state. The client-side
@@ -224,28 +240,18 @@ export class SavedObjectsClient {
       this.batchResolveQueue = [];
 
       try {
-        const objectsToFetch = getObjectsToFetch(queue);
-        const { resolved_objects: savedObjects } = await this.performBulkResolve(objectsToFetch);
-
-        queue.forEach((queueItem) => {
-          const foundObject = savedObjects.find((resolveResponse) => {
-            return (
-              resolveResponse.saved_object.id === queueItem.id &&
-              resolveResponse.saved_object.type === queueItem.type
-            );
-          });
-
-          if (foundObject) {
-            // multiple calls may have been requested the same object.
-            // we need to clone to avoid sharing references between the instances
-            queueItem.resolve(this.createResolvedSavedObject(cloneDeep(foundObject)));
-          } else {
-            queueItem.resolve(
-              this.createResolvedSavedObject({
-                saved_object: pick(queueItem, ['id', 'type']),
-              } as SavedObjectsResolveResponse)
-            );
-          }
+        const { objectsToResolve, responseIndices } = getObjectsToResolve(queue);
+        const { resolved_objects: resolvedObjects } = await this.performBulkResolve(
+          objectsToResolve
+        );
+
+        queue.forEach((queueItem, i) => {
+          // This differs from the older processBatchGetQueue approach because the resolved object IDs are *not* guaranteed to be the same.
+          // Instead, we rely on the guarantee that the objects in the bulkResolve response will be in the same order as the requests.
+          // However, we still need to clone the response object because we deduplicate batched requests.
+          const responseIndex = responseIndices[i];
+          const clone = cloneDeep(resolvedObjects[responseIndex]);
+          queueItem.resolve(this.createResolvedSavedObject(clone));
         });
       } catch (err) {
         queue.forEach((queueItem) => {

src/core/server/saved_objects/migrationsv2/actions/integration_tests/actions.test.ts

@@ -422,8 +422,7 @@ describe('migration actions', () => {
 
   // Reindex doesn't return any errors on it's own, so we have to test
   // together with waitForReindexTask
-  // FLAKY: https://github.com/elastic/kibana/issues/103231
-  describe.skip('reindex & waitForReindexTask', () => {
+  describe('reindex & waitForReindexTask', () => {
     it('resolves right when reindex succeeds without reindex script', async () => {
       const res = (await reindex({
         client,
@@ -792,6 +791,11 @@ describe('migration actions', () => {
               `);
     });
     it('resolves left wait_for_task_completion_timeout when the task does not finish within the timeout', async () => {
+      await waitForIndexStatusYellow({
+        client,
+        index: '.kibana_1',
+      })();
+
       const res = (await reindex({
         client,
         sourceIndex: '.kibana_1',

src/dev/license_checker/config.ts

@@ -73,9 +73,8 @@ export const DEV_ONLY_LICENSE_ALLOWED = ['MPL-2.0'];
 export const LICENSE_OVERRIDES = {
   'jsts@1.6.2': ['Eclipse Distribution License - v 1.0'], // cf. https://github.com/bjornharrtell/jsts
   '@mapbox/jsonlint-lines-primitives@2.0.2': ['MIT'], // license in readme https://github.com/tmcw/jsonlint
-  'node-sql-parser@3.6.1': ['(GPL-2.0 OR MIT)'], // GPL-2.0* https://github.com/taozhi8833998/node-sql-parser
   '@elastic/ems-client@7.15.0': ['Elastic License 2.0'],
-  '@elastic/eui@37.6.0': ['SSPL-1.0 OR Elastic License 2.0'],
+  '@elastic/eui@38.0.1': ['SSPL-1.0 OR Elastic License 2.0'],
 
   // TODO can be removed if the https://github.com/jindw/xmldom/issues/239 is released
   'xmldom@0.1.27': ['MIT'],

src/plugins/kibana_usage_collection/server/collectors/management/schema.ts

@@ -408,12 +408,12 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'text',
     _meta: { description: 'Non-default value of setting.' },
   },
-  'apm:enableSignificantTerms': {
+  'observability:enableInspectEsQueries': {
     type: 'boolean',
     _meta: { description: 'Non-default value of setting.' },
   },
-  'observability:enableInspectEsQueries': {
-    type: 'boolean',
+  'observability:maxSuggestions': {
+    type: 'integer',
     _meta: { description: 'Non-default value of setting.' },
   },
   'banners:placement': {

src/plugins/kibana_usage_collection/server/collectors/management/types.ts

@@ -34,8 +34,8 @@ export interface UsageStats {
   'discover:showMultiFields': boolean;
   'discover:maxDocFieldsDisplayed': number;
   'securitySolution:rulesTableRefresh': string;
-  'apm:enableSignificantTerms': boolean;
   'observability:enableInspectEsQueries': boolean;
+  'observability:maxSuggestions': number;
   'visualize:enableLabs': boolean;
   'visualization:heatmap:maxBuckets': number;
   'visualization:colorMapping': string;

src/plugins/telemetry/schema/oss_plugins.json

@@ -7629,14 +7629,14 @@
             "description": "Non-default value of setting."
           }
         },
-        "apm:enableSignificantTerms": {
+        "observability:enableInspectEsQueries": {
           "type": "boolean",
           "_meta": {
             "description": "Non-default value of setting."
           }
         },
-        "observability:enableInspectEsQueries": {
-          "type": "boolean",
+        "observability:maxSuggestions": {
+          "type": "integer",
           "_meta": {
             "description": "Non-default value of setting."
           }

typings/js_sql_parser.d.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+declare module 'js-sql-parser';

x-pack/plugins/apm/common/anomaly_detection.ts

@@ -33,6 +33,8 @@ export function getSeverityColor(score: number) {
   return mlGetSeverityColor(score);
 }
 
+export const ML_TRANSACTION_LATENCY_DETECTOR_INDEX = 0;
+
 export const ML_ERRORS = {
   INVALID_LICENSE: i18n.translate(
     'xpack.apm.anomaly_detection.error.invalid_license',