Merge branch '8.16' into backport/8.16/pr-193812

.buildkite/ftr_platform_stateful_configs.yml

@@ -249,6 +249,7 @@ enabled:
   - x-pack/test/functional/apps/ml/permissions/config.ts
   - x-pack/test/functional/apps/ml/short_tests/config.ts
   - x-pack/test/functional/apps/ml/stack_management_jobs/config.ts
+  - x-pack/test/functional/apps/ml/memory_usage/config.ts
   - x-pack/test/functional/apps/monitoring/config.ts
   - x-pack/test/functional/apps/painless_lab/config.ts
   - x-pack/test/functional/apps/remote_clusters/config.ts

docs/user/dashboard/lens.asciidoc

@@ -371,7 +371,7 @@ The following component menus are available:
 
 When creating or editing a visualization, you can customize the way the legend gets displayed, and the data it displays. To do that, look for the image:images/legend-icon.svg[Legend icon] icon.
 
-image::images/legend-popover.png[Menu with options to customize the legend of a visualization, width=40%]
+image::images/legend-popover-8.16.0.png[Menu with options to customize the legend of a visualization, width=50%]
 
 NOTE: The options available can vary based on the type of chart you're setting up. For example, showing additional statistics is only possible for time series charts.
 

docs/user/dashboard/tutorial-create-a-dashboard-of-lens-panels.asciidoc

@@ -60,7 +60,7 @@ To create the visualizations in this tutorial, you'll use the following fields:
 Click a field name to view more details, such as its top values and distribution.
 
 [role="screenshot"]
-image::https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/bltff29bc11413cc5d8/66fdc386762d3a15135abec4/tutorial-field-top-values-distribution.gif[Clicking a field name to view more details]
+image::https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/blt25c7a05e0f5daaa3/671966b9b1d3dd705a2ae650/tutorial-field-more-info.gif[Clicking a field name to view more details]
 
 [discrete]
 [[view-the-number-of-website-visitors]]
@@ -73,12 +73,12 @@ The only number function that you can use with *clientip* is *Unique count*, als
 . Open the *Visualization type* dropdown, then select *Metric*.
 +
 [role="screenshot"]
-image::images/lens_visualizationTypeDropdown_8.4.0.png[Visualization type dropdown]
+image::images/visualization-type-dropdown-8.16.0.png[Visualization type dropdown]
 
 . From the *Available fields* list, drag *clientip* to the workspace or layer pane.
 +
 [role="screenshot"]
-image::images/lens_metricUniqueCountOfClientip_8.4.0.png[Metric visualization of the clientip field]
+image::images/tutorial-unique-count-of-client-ip-8.16.0.png[Metric visualization of the clientip field]
 +
 In the layer pane, *Unique count of clientip* appears because the editor automatically applies the *Unique count* function to the *clientip* field. *Unique count* is the only numeric function that works with IP addresses.
 
@@ -133,12 +133,12 @@ To save space on the dashboard, hide the axis labels.
 . Open the *Left axis* menu, then select *None* from the *Axis title* dropdown.
 +
 [role="screenshot"]
-image::images/lens_lineChartMetricOverTimeLeftAxis_8.7.png[Left axis menu]
+image::images/line-chart-left-axis-8.16.0.png[Left axis menu, width=50%]
 
 . Open the *Bottom axis* menu, then select *None* from the *Axis title* dropdown.
 +
 [role="screenshot"]
-image::images/lens_lineChartMetricOverTimeBottomAxis_8.7.png[Bottom axis menu]
+image::images/line-chart-bottom-axis-8.16.0.png[Bottom axis menu, width=50%]
 
 . Click *Save and return*
 
@@ -180,7 +180,7 @@ The chart labels are unable to display because the *request.keyword* field conta
 . Open the *Visualization type* dropdown, then select *Table*.
 +
 [role="screenshot"]
-image::images/lens_end_to_end_2_1_2.png[Table with top values of request.keyword by most unique visitors]
+image::images/table-with-request-keyword-and-client-ip-8.16.0.png[Table with top values of request.keyword by most unique visitors, width=70%]
 
 . In the layer pane, click *Top 5 values of request.keyword*.
 

docs/user/plugins.asciidoc

@@ -18,7 +18,8 @@ Plugin developers must release a new version of their plugin for each new {kib}
 == Known plugins
 
 The known plugins were tested for {kib} *5.x*, so we are unable to guarantee compatibility with your version of {kib}. The {kib} installer rejects any plugins that haven't been published for your specific version of {kib}. 
-We are unable to evaluate or maintain the known plugins, so care should be taken before installation.
+
+IMPORTANT: Known plugins are developed and maintained outside of Elastic. They are not supported by Elastic. If you encounter an issue with a community plugin, contact the plugin's owner.
 
 [float]
 === Apps
@@ -182,4 +183,4 @@ you must specify the path to that configuration file each time you use the `bin/
 0:: Success
 64:: Unknown command or incorrect option parameter
 74:: I/O error
-70:: Other error
+70:: Other error

fleet_packages.json

@@ -34,7 +34,7 @@
   },
   {
     "name": "endpoint",
-    "version": "8.15.1"
+    "version": "8.16.0"
   },
   {
     "name": "fleet_server",

oas_docs/README.md

@@ -1,9 +1,52 @@
-The `bundle.json` and `bundle.serverless.json` files are generated automatically.
-See `node scripts/capture_oas_snapshot --help` for more info.
+# Kibana API reference documentation
 
-The `output/kibana.serverless.yaml` and `output/kibana.yaml` files join some manually-maintained files with the automatically generated files.
-To add integrate more files into this bundle, edit the appropriate `oas_docs/scripts/merge*.js` files.
-To generate the bundled files, run `make api-docs` (or `make api-docs-serverless` and `make api-docs-stateful`).
-To lint them, run `make api-docs-lint` (or `make api-docs-lint-serverless` and `make api-lint-stateful`).
+Documentation about our OpenAPI bundling workflow and configuration. See Kibana's hosted [stateful](https://www.elastic.co/docs/api/doc/kibana) and [serverless](https://www.elastic.co/docs/api/doc/serverless) docs.
 
-To apply some overlays that perform some post-processing and append some content, run `make api-docs-overlay`.
+## Workflow
+
+The final goal of this workflow is to produce an OpenAPI bundle containing all Kibana's public APIs.
+
+### Step 0
+
+OAS from Kibana's APIs are continuously extracted and captured in `bundle.json` and `bundle.serverless.json` as fully formed OAS documentation. See `node scripts/capture_oas_snapshot --help` for more info.
+
+These bundles form the basis of our OpenAPI bundles to which we append and layer extra information before publishing.
+
+### Step 1
+
+Append pre-existing bundles not extracted from code using [`kbn-openapi-bundler`](../packages/kbn-openapi-bundler/README.md) to produce the final resulting bundles.
+
+To add more files into the final bundle, edit the appropriate `oas_docs/scripts/merge*.js` files.
+
+### Step 2
+
+Apply any final overalys to the document that might include examples or final tweaks (see the ["Scripts"](#scripts) section for more details).
+
+## Scripts
+
+The `oas_docs/scripts` folder contains scripts that point to the source domain-specific OpenAPI bundles and specify additional parameters for producing the final output bundle. Currently, there are the following scripts:
+
+- `merge_ess_oas.js` script produces production an output bundle for ESS
+
+- `merge_serverless_oas.js` script produces production an output bundle for Serverless
+
+### Output Kibana OpenAPI bundles
+
+The `oas_docs/output` folder contains the final resulting Kibana OpenAPI bundles
+
+- `kibana.yaml` production ready ESS OpenAPI bundle
+- `kibana.serverless.yaml` production ready Serverless OpenAPI bundle
+
+## Bundling commands
+
+Besides the scripts in the `oas_docs/scripts` folder, there is an `oas_docs/makefile` to simplify the workflow. The following makefile targets are available:
+
+| Command                    | Description                                                                                                                                                                                                        |
+| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `api-docs`                 | Builds ESS Kibana OpenAPI bundle                                                                                                                                                                                   |
+| `api-docs-serverless`      | Builds Serverless Kibana OpenAPI bundle                                                                                                                                                                            |
+| `api-docs-lint`            | Lints built result bundles                                                                                                                                                                                         |
+| `api-docs-lint-errs`       | Lints built result bundles for errors                                                                                                                                                                              |
+| `api-docs-preview`         | Generates (ESS + Serverless) Kibana OpenAPI bundles preview                                                                                                                                                        |
+| `api-docs-overlay`         | Applies [overlays](https://docs.bump.sh/help/specification-support/overlays/) from `overlays` folder to the Kibana OpenAPI bundles and generate `*.new.yaml` files. Overlays help to fine tune the result bundles. |
+| `api-docs-overlay-preview` | Generates a preview for bundles produced by `api-docs-overlay`                                                                                                                                                     |

package.json

@@ -113,20 +113,20 @@
     "@elastic/apm-rum": "^5.16.1",
     "@elastic/apm-rum-core": "^5.21.1",
     "@elastic/apm-rum-react": "^2.0.3",
-    "@elastic/charts": "68.0.0",
+    "@elastic/charts": "68.0.2",
     "@elastic/datemath": "5.0.3",
     "@elastic/ebt": "^1.1.1",
     "@elastic/ecs": "^8.11.1",
     "@elastic/elasticsearch": "^8.15.0",
     "@elastic/ems-client": "8.5.3",
     "@elastic/eui": "97.0.0-backport.0",
     "@elastic/filesaver": "1.1.2",
-    "@elastic/node-crypto": "1.2.1",
+    "@elastic/node-crypto": "^1.2.3",
     "@elastic/numeral": "^2.5.1",
     "@elastic/react-search-ui": "^1.20.2",
     "@elastic/react-search-ui-views": "^1.20.2",
     "@elastic/request-converter": "^8.15.4",
-    "@elastic/request-crypto": "2.0.2",
+    "@elastic/request-crypto": "^2.0.3",
     "@elastic/search-ui": "^1.20.2",
     "@elastic/search-ui-app-search-connector": "^1.20.2",
     "@elastic/search-ui-engines-connector": "^1.20.2",

packages/content-management/content_insights/content_insights_public/src/components/activity_view.tsx

@@ -9,6 +9,7 @@
 
 import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSpacer, EuiText } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
+import moment from 'moment-timezone';
 import { FormattedMessage } from '@kbn/i18n-react';
 import React from 'react';
 import {
@@ -97,10 +98,16 @@ export const ActivityView = ({ item }: ActivityViewProps) => {
   );
 };
 
-const dateFormatter = new Intl.DateTimeFormat(i18n.getLocale(), {
-  dateStyle: 'long',
-  timeStyle: 'short',
-});
+const formatDate = (time: string) => {
+  const locale = i18n.getLocale();
+  const timeZone = moment().tz();
+
+  return new Intl.DateTimeFormat(locale, {
+    dateStyle: 'long',
+    timeStyle: 'short',
+    timeZone,
+  }).format(new Date(time));
+};
 
 const ActivityCard = ({
   what,
@@ -130,7 +137,7 @@ const ActivityCard = ({
               id="contentManagement.contentEditor.activity.lastUpdatedByDateTime"
               defaultMessage="on {dateTime}"
               values={{
-                dateTime: dateFormatter.format(new Date(when)),
+                dateTime: formatDate(when),
               }}
             />
           </EuiText>

packages/core/http/core-http-server-internal/src/static_assets/util.ts

@@ -14,11 +14,23 @@ function isEmptyPathname(pathname: string): boolean {
 }
 
 function removeTailSlashes(pathname: string): string {
-  return pathname.replace(/\/+$/, '');
+  let updated = pathname;
+
+  while (updated.endsWith('/')) {
+    updated = updated.substring(0, updated.length - 1);
+  }
+
+  return updated;
 }
 
 function removeLeadSlashes(pathname: string): string {
-  return pathname.replace(/^\/+/, '');
+  let updated = pathname;
+
+  while (updated.startsWith('/')) {
+    updated = updated.substring(1);
+  }
+
+  return updated;
 }
 
 export function removeSurroundingSlashes(pathname: string): string {

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.test.ts

@@ -176,8 +176,9 @@ describe('checkTargetTypesMappings', () => {
 
             const result = await task();
             expect(result).toEqual(
-              Either.right({
-                type: 'types_match' as const,
+              Either.left({
+                type: 'types_added' as const,
+                newTypes: ['type3'],
               })
             );
           });

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.ts

@@ -11,7 +11,7 @@ import * as Either from 'fp-ts/lib/Either';
 import * as TaskEither from 'fp-ts/lib/TaskEither';
 
 import type { IndexMapping, VirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal';
-import { getUpdatedTypes } from '../core/compare_mappings';
+import { getNewAndUpdatedTypes } from '../core/compare_mappings';
 
 /** @internal */
 export interface CheckTargetTypesMappingsParams {
@@ -38,6 +38,12 @@ export interface TypesChanged {
   updatedTypes: string[];
 }
 
+/** @internal */
+export interface TypesAdded {
+  type: 'types_added';
+  newTypes: string[];
+}
+
 export const checkTargetTypesMappings =
   ({
     indexTypes,
@@ -46,7 +52,7 @@ export const checkTargetTypesMappings =
     latestMappingsVersions,
     hashToVersionMap = {},
   }: CheckTargetTypesMappingsParams): TaskEither.TaskEither<
-    IndexMappingsIncomplete | TypesChanged,
+    IndexMappingsIncomplete | TypesChanged | TypesAdded,
     TypesMatch
   > =>
   async () => {
@@ -58,7 +64,7 @@ export const checkTargetTypesMappings =
       return Either.left({ type: 'index_mappings_incomplete' as const });
     }
 
-    const updatedTypes = getUpdatedTypes({
+    const { newTypes, updatedTypes } = getNewAndUpdatedTypes({
       indexTypes,
       indexMeta: indexMappings?._meta,
       latestMappingsVersions,
@@ -70,6 +76,11 @@ export const checkTargetTypesMappings =
         type: 'types_changed' as const,
         updatedTypes,
       });
+    } else if (newTypes.length) {
+      return Either.left({
+        type: 'types_added' as const,
+        newTypes,
+      });
     } else {
       return Either.right({ type: 'types_match' as const });
     }

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/es_errors.test.ts

@@ -8,6 +8,7 @@
  */
 
 import {
+  hasAllKeywordsInOrder,
   isClusterShardLimitExceeded,
   isIncompatibleMappingException,
   isIndexNotFoundException,
@@ -128,3 +129,31 @@ describe('isClusterShardLimitExceeded', () => {
     expect(isClusterShardLimitExceeded(undefined)).toEqual(false);
   });
 });
+
+describe('hasAllKeywordsInOrder', () => {
+  it('returns false if not all keywords are present', () => {
+    expect(
+      hasAllKeywordsInOrder('some keywords in a message', ['some', 'in', 'message', 'missing'])
+    ).toEqual(false);
+  });
+
+  it('returns false if keywords are not in the right order', () => {
+    expect(
+      hasAllKeywordsInOrder('some keywords in a message', ['some', 'message', 'keywords'])
+    ).toEqual(false);
+  });
+
+  it('returns false if the message is empty', () => {
+    expect(hasAllKeywordsInOrder('', ['some', 'message', 'keywords'])).toEqual(false);
+  });
+
+  it('returns false if the keyword list is empty', () => {
+    expect(hasAllKeywordsInOrder('some keywords in a message', [])).toEqual(false);
+  });
+
+  it('returns true if keywords are present and in the right order', () => {
+    expect(
+      hasAllKeywordsInOrder('some keywords in a message', ['some', 'keywords', 'in', 'message'])
+    ).toEqual(true);
+  });
+});

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/es_errors.ts

@@ -7,34 +7,46 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { ErrorCause } from '@elastic/elasticsearch/lib/api/types';
 
-export const isWriteBlockException = (errorCause?: estypes.ErrorCause): boolean => {
+export const isWriteBlockException = (errorCause?: ErrorCause): boolean => {
   return (
     errorCause?.type === 'cluster_block_exception' &&
-    errorCause?.reason?.match(/index \[.+] blocked by: \[FORBIDDEN\/8\/.+ \(api\)\]/) !== null
+    hasAllKeywordsInOrder(errorCause?.reason, ['index [', '] blocked by: [FORBIDDEN/8/', ' (api)]'])
   );
 };
 
-export const isIncompatibleMappingException = (errorCause?: estypes.ErrorCause): boolean => {
+export const isIncompatibleMappingException = (errorCause?: ErrorCause): boolean => {
   return (
     errorCause?.type === 'strict_dynamic_mapping_exception' ||
     errorCause?.type === 'mapper_parsing_exception' ||
     errorCause?.type === 'document_parsing_exception'
   );
 };
 
-export const isIndexNotFoundException = (errorCause?: estypes.ErrorCause): boolean => {
+export const isIndexNotFoundException = (errorCause?: ErrorCause): boolean => {
   return errorCause?.type === 'index_not_found_exception';
 };
 
-export const isClusterShardLimitExceeded = (errorCause?: estypes.ErrorCause): boolean => {
+export const isClusterShardLimitExceeded = (errorCause?: ErrorCause): boolean => {
   // traditional ES: validation_exception. serverless ES: illegal_argument_exception
   return (
     (errorCause?.type === 'validation_exception' ||
       errorCause?.type === 'illegal_argument_exception') &&
-    errorCause?.reason?.match(
-      /this action would add .* shards, but this cluster currently has .* maximum normal shards open/
-    ) !== null
+    hasAllKeywordsInOrder(errorCause?.reason, [
+      'this action would add',
+      'shards, but this cluster currently has',
+      'maximum normal shards open',
+    ])
   );
 };
+
+export const hasAllKeywordsInOrder = (message: string | undefined, keywords: string[]): boolean => {
+  if (!message || !keywords.length) {
+    return false;
+  }
+
+  const keywordIndices = keywords.map((keyword) => message?.indexOf(keyword) ?? -1);
+  // check that all keywords are present and in the right order
+  return keywordIndices.every((v, i, a) => v >= 0 && (!i || a[i - 1] <= v));
+};