Merge branch 'main' into d/2021-11-15-optimize-search-bar-react-rende…

…ring

.backportrc.json

@@ -1,7 +1,8 @@
 {
-  "upstream": "elastic/kibana",
+  "repoOwner": "elastic",
+  "repoName": "kibana",
   "targetBranchChoices": [
-    { "name": "main", "checked": true },
+    "main",
     "8.0",
     "7.17",
     "7.16",
@@ -32,7 +33,9 @@
     "6.0",
     "5.6"
   ],
-  "targetPRLabels": ["backport"],
+  "targetPRLabels": [
+    "backport"
+  ],
   "branchLabelMapping": {
     "^v8.1.0$": "main",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"

.buildkite/scripts/common/setup_node.sh

@@ -61,7 +61,14 @@ YARN_VERSION=$(node -e "console.log(String(require('./package.json').engines.yar
 export YARN_VERSION
 
 if [[ ! $(which yarn) || $(yarn --version) != "$YARN_VERSION" ]]; then
-  npm install -g "yarn@^${YARN_VERSION}"
+  rm -rf "$(npm root -g)/yarn" # in case the directory is in a bad state
+  if [[ ! $(npm install -g "yarn@^${YARN_VERSION}") ]]; then
+    # If this command is terminated early, e.g. because the build was cancelled in buildkite,
+    # a yarn directory is left behind in a bad state that can cause all subsequent installs to fail
+    rm -rf "$(npm root -g)/yarn"
+    echo "Trying again to install yarn..."
+    npm install -g "yarn@^${YARN_VERSION}"
+  fi
 fi
 
 yarn config set yarn-offline-mirror "$YARN_OFFLINE_CACHE"

.buildkite/scripts/lifecycle/pre_command.sh

@@ -9,8 +9,22 @@ export BUILDKITE_TOKEN
 
 echo '--- Install buildkite dependencies'
 cd '.buildkite'
-retry 5 15 yarn install --production --pure-lockfile
-cd -
+
+# If this yarn install is terminated early, e.g. if the build is cancelled in buildkite,
+# A node module could end up in a bad state that can cause all future builds to fail
+# So, let's cache clean and try again to make sure that's not what caused the error
+install_deps() {
+  yarn install --production --pure-lockfile
+  EXIT=$?
+  if [[ "$EXIT" != "0" ]]; then
+    yarn cache clean
+  fi
+  return $EXIT
+}
+
+retry 5 15 install_deps
+
+cd ..
 
 node .buildkite/scripts/lifecycle/print_agent_links.js || true
 

.github/CODEOWNERS

@@ -365,28 +365,28 @@
 /x-pack/plugins/enterprise_search/server/collectors/workplace_search/ @elastic/workplace-search-frontend
 /x-pack/plugins/enterprise_search/server/saved_objects/workplace_search/ @elastic/workplace-search-frontend
 
-# Stack Management
-/src/plugins/dev_tools/ @elastic/kibana-stack-management
-/src/plugins/console/  @elastic/kibana-stack-management
-/src/plugins/es_ui_shared/  @elastic/kibana-stack-management
-/src/plugins/management/ @elastic/kibana-stack-management
-/x-pack/plugins/cross_cluster_replication/  @elastic/kibana-stack-management
-/x-pack/plugins/index_lifecycle_management/  @elastic/kibana-stack-management
-/x-pack/plugins/grokdebugger/  @elastic/kibana-stack-management
-/x-pack/plugins/index_management/  @elastic/kibana-stack-management
-/x-pack/plugins/license_api_guard/  @elastic/kibana-stack-management
-/x-pack/plugins/license_management/  @elastic/kibana-stack-management
-/x-pack/plugins/painless_lab/  @elastic/kibana-stack-management
-/x-pack/plugins/remote_clusters/  @elastic/kibana-stack-management
-/x-pack/plugins/rollup/  @elastic/kibana-stack-management
-/x-pack/plugins/searchprofiler/  @elastic/kibana-stack-management
-/x-pack/plugins/snapshot_restore/  @elastic/kibana-stack-management
-/x-pack/plugins/upgrade_assistant/  @elastic/kibana-stack-management
-/x-pack/plugins/watcher/  @elastic/kibana-stack-management
-/x-pack/plugins/ingest_pipelines/  @elastic/kibana-stack-management
-/packages/kbn-ace/ @elastic/kibana-stack-management
-/packages/kbn-monaco/ @elastic/kibana-stack-management
-#CC# /x-pack/plugins/cross_cluster_replication/ @elastic/kibana-stack-management
+# Management Experience - Deployment Management
+/src/plugins/dev_tools/ @elastic/platform-deployment-management
+/src/plugins/console/  @elastic/platform-deployment-management
+/src/plugins/es_ui_shared/  @elastic/platform-deployment-management
+/src/plugins/management/ @elastic/platform-deployment-management
+/x-pack/plugins/cross_cluster_replication/  @elastic/platform-deployment-management
+/x-pack/plugins/index_lifecycle_management/  @elastic/platform-deployment-management
+/x-pack/plugins/grokdebugger/  @elastic/platform-deployment-management
+/x-pack/plugins/index_management/  @elastic/platform-deployment-management
+/x-pack/plugins/license_api_guard/  @elastic/platform-deployment-management
+/x-pack/plugins/license_management/  @elastic/platform-deployment-management
+/x-pack/plugins/painless_lab/  @elastic/platform-deployment-management
+/x-pack/plugins/remote_clusters/  @elastic/platform-deployment-management
+/x-pack/plugins/rollup/  @elastic/platform-deployment-management
+/x-pack/plugins/searchprofiler/  @elastic/platform-deployment-management
+/x-pack/plugins/snapshot_restore/  @elastic/platform-deployment-management
+/x-pack/plugins/upgrade_assistant/  @elastic/platform-deployment-management
+/x-pack/plugins/watcher/  @elastic/platform-deployment-management
+/x-pack/plugins/ingest_pipelines/  @elastic/platform-deployment-management
+/packages/kbn-ace/ @elastic/platform-deployment-management
+/packages/kbn-monaco/ @elastic/platform-deployment-management
+#CC# /x-pack/plugins/cross_cluster_replication/ @elastic/platform-deployment-management
 
 # Security Solution
 /x-pack/test/endpoint_api_integration_no_ingest/ @elastic/security-solution

docs/settings/fleet-settings.asciidoc

@@ -10,7 +10,7 @@
 In {ecloud}, {fleet} flags are already configured.
 ====
 
-You can configure `xpack.fleet` settings in your `kibana.yml`. 
+You can configure `xpack.fleet` settings in your `kibana.yml`.
 By default, {fleet} is enabled. To use {fleet}, you also need to configure {kib} and {es} hosts.
 
 See the {fleet-guide}/index.html[{fleet}] docs for more information.
@@ -21,7 +21,7 @@ See the {fleet-guide}/index.html[{fleet}] docs for more information.
 [cols="2*<"]
 |===
 | `xpack.fleet.agents.enabled` {ess-icon}
-  | Set to `true` (default) to enable {fleet}. 
+  | Set to `true` (default) to enable {fleet}.
 |===
 
 [[fleet-data-visualizer-settings]]
@@ -33,7 +33,8 @@ See the {fleet-guide}/index.html[{fleet}] docs for more information.
 | `xpack.fleet.registryUrl`
   | The address to use to reach the {package-manager} registry.
 | `xpack.fleet.registryProxyUrl`
-  | The proxy address to use to reach the {package-manager} registry.
+  | The proxy address to use to reach the {package-manager} registry if an internet connection is not directly available.
+  Refer to {fleet-guide}/air-gapped.html[Air-gapped environments] for details.
 
 |===
 
@@ -62,7 +63,7 @@ want {fleet} to load up by default.
 | `xpack.fleet.packages`
   | List of integrations that are installed when the {fleet} app starts
   up for the first time. Required properties are:
-  
+
   `name`:: Name of the integration from the package registry.
   `version`:: Either an exact semantic version, or the keyword `latest` to fetch
   the latest integration version.
@@ -75,7 +76,7 @@ Required properties are:
   `name`:: Policy name.
 
 Optional properties are:
-  
+
   `description`:: Text description of this policy.
   `namespace`:: String identifying this policy's namespace.
   `monitoring_enabled`:: List of keywords that specify the monitoring data to collect.

docs/settings/reporting-settings.asciidoc

@@ -50,7 +50,7 @@ xpack.reporting.encryptionKey: "something_secret"
 [[reporting-kibana-server-settings]]
 ==== {kib} server settings
 
-Reporting opens the {kib} web interface in a server process to generate
+For PNG and PDF reports, Reporting opens the {kib} web interface in a headless server process to generate
 screenshots of {kib} visualizations. In most cases, the default settings
 work and you don't need to configure the {report-features} to communicate with {kib}.
 
@@ -129,14 +129,11 @@ If capturing a report fails for any reason, {kib} will re-attempt other reportin
 `xpack.reporting.capture.loadDelay`::
 Specify the {time-units}[amount of time] before taking a screenshot when visualizations are not evented. All visualizations that ship with {kib} are evented, so this setting should not have much effect. If you are seeing empty images instead of visualizations, try increasing this value. Defaults to `3s`.
 
-[[xpack-reporting-browser]] `xpack.reporting.capture.browser.type` {ess-icon}::
-Specifies the browser to use to capture screenshots. This setting exists for backward compatibility. The only valid option is `chromium`.
-
 [float]
 [[reporting-chromium-settings]]
 ==== Chromium settings
 
-When <<xpack-reporting-browser, `xpack.reporting.capture.browser.type`>> is set to `chromium` (default) you can also specify the following settings.
+For PDF and PNG reports, Reporting spawns a headless Chromium browser process on the server to load and capture a screenshot of the {kib} app. When installing {kib} on Linux and Windows platforms, the Chromium binary comes bundled with the {kib} download. For Mac platforms, the Chromium binary is downloaded the first time {kib} is started.
 
 `xpack.reporting.capture.browser.chromium.disableSandbox`::
 It is recommended that you research the feasibility of enabling unprivileged user namespaces. An exception is if you are running {kib} in Docker because the container runs in a user namespace with the built-in seccomp/bpf filters. For more information, refer to <<reporting-chromium-sandbox>>. Defaults to `false` for all operating systems except Debian and Red Hat Linux, which use `true`.

docs/user/production-considerations/reporting-production-considerations.asciidoc

@@ -8,7 +8,15 @@
 :keywords: administrator, analyst, concept, setup, reporting
 :description: Consider the production components that are used to generate reports.
 
-To generate reports, {kib} uses a custom build of the Chromium web browser, which runs on the {kib} server in headless mode to load {kib} and capture the rendered {kib} visualizations as images. Chromium is an open-source project not related to Elastic, but the Chromium binary for {kib} has been custom-built by Elastic to make sure it works with minimal setup. The operating system that the {kib} server uses can require additional dependencies for Chromium.
+To generate reports, {kib} uses the Chromium web browser, which runs on the server in headless mode. Chromium is an open-source project not related to Elastic, and is embedded into {kib}. Chromium may require additional OS dependencies to run properly.
+
+[NOTE]
+============
+Chromium is not embedded into {kib} for the Darwin (Mac OS) architecture. When
+running on Darwin, Reporting will download Chromium into the proper area of
+the {kib} installation path the first time the server starts. To separately
+download and install the browser, see <<reporting-manual-chromium-install>>.
+============
 
 [float]
 [[reporting-chromium-sandbox]]

docs/user/reporting/reporting-troubleshooting.asciidoc

@@ -14,8 +14,8 @@ Having trouble? Here are solutions to common problems you might encounter while
 * <<reporting-troubleshooting-error-messages>>
 * <<reporting-troubleshooting-puppeteer-debug-logs>>
 * <<reporting-troubleshooting-system-requirements>>
-* <<reporting-troubleshooting-arm-systems>>
 * <<reporting-troubleshooting-maps-ems>>
+* <<reporting-manual-chromium-install>>
 
 [float]
 [[reporting-diagnostics]]
@@ -150,12 +150,6 @@ requests to render.
 If the {kib} instance doesn't have enough memory to run the report, the report fails with an error such as `Error: Page crashed!`
 In this case, try increasing the memory for the {kib} instance to 2GB.
 
-[float]
-[[reporting-troubleshooting-arm-systems]]
-=== ARM systems
-
-Chromium is not compatible with ARM RHEL.
-
 [float]
 [[reporting-troubleshooting-maps-ems]]
 === Unable to connect to Elastic Maps Service
@@ -164,3 +158,24 @@ https://www.elastic.co/elastic-maps-service[{ems} ({ems-init})] is a service tha
 tile layers and vector shapes of administrative boundaries.
 If a report contains a map with a missing basemap layer or administrative boundary, the {kib} server does not have access to {ems-init}.
 See <<maps-connect-to-ems>> for information on how to connect your {kib} server to {ems-init}.
+
+[float]
+[[reporting-manual-chromium-install]]
+=== Manually install the Chromium browser for Darwin
+Chromium is not embedded into {kib} for the Darwin (Mac OS) architecture. When
+running {kib} on Darwin, Reporting will download Chromium into the proper area of
+the {kib} installation path the first time the server starts. If the server
+does not have access to the Internet, you must download the
+Chromium browser and install it into the {kib} installation path.
+
+1. Download the Chromium zip file:
+
+** For https://commondatastorage.googleapis.com/chromium-browser-snapshots/Mac/901912/chrome-mac.zip[x64] systems
+** For https://commondatastorage.googleapis.com/chromium-browser-snapshots/Mac_Arm/901913/chrome-mac.zip[ARM] systems
+
+2. Copy the zip file into the holding area. Relative to the root directory of {kib}, the path is:
+
+** `.chromium/x64` for x64 systems
+** `.chromium/arm64` for ARM systems
+
+When {kib} starts, it will automatically extract the browser from the zip file, and is then ready for PNG and PDF reports.

package.json

@@ -711,7 +711,7 @@
     "babel-plugin-require-context-hook": "^1.0.0",
     "babel-plugin-styled-components": "^2.0.2",
     "babel-plugin-transform-react-remove-prop-types": "^0.4.24",
-    "backport": "^5.6.6",
+    "backport": "^6.1.1",
     "callsites": "^3.1.0",
     "chai": "3.5.0",
     "chance": "1.0.18",

scripts/backport.js

@@ -13,4 +13,4 @@ var process = require('process');
 var args = process.argv.slice(2);
 
 var backport = require('backport');
-backport.run({}, args);
+backport.backportRun({}, args);

src/dev/build/tasks/install_chromium.js

@@ -7,18 +7,27 @@
  */
 
 // eslint-disable-next-line @kbn/eslint/no-restricted-paths
-import { install } from '../../../../x-pack/plugins/screenshotting/server/utils';
+import { install, paths } from '../../../../x-pack/plugins/screenshotting/server/utils';
 
 export const InstallChromium = {
   description: 'Installing Chromium',
 
   async run(config, log, build) {
+    const preInstalledPackages = paths.packages.filter((p) => p.isPreInstalled);
+
     for (const platform of config.getNodePlatforms()) {
+      const pkg = paths.find(platform.getName(), platform.getArchitecture(), preInstalledPackages);
       const target = `${platform.getName()}-${platform.getArchitecture()}`;
-      log.info(`Installing Chromium for ${target}`);
 
-      // revert after https://github.com/elastic/kibana/issues/109949
-      if (target === 'darwin-arm64') continue;
+      if (!pkg) {
+        log.info(`Skipping Chromium install for ${target}`);
+
+        // Unbundled chromium packages (for Darwin): Chromium is downloaded at
+        // server startup, rather than being pre-installed
+        continue;
+      }
+
+      log.info(`Installing Chromium for ${target}`);
 
       const logger = {
         get: log.withType.bind(log),
@@ -31,12 +40,8 @@ export const InstallChromium = {
         log: log.write.bind(log),
       };
 
-      await install(
-        logger,
-        build.resolvePathForPlatform(platform, 'x-pack/plugins/screenshotting/chromium'),
-        platform.getName(),
-        platform.getArchitecture()
-      );
+      const path = build.resolvePathForPlatform(platform, 'x-pack/plugins/screenshotting/chromium');
+      await install(logger, pkg, path);
     }
   },
 };

src/dev/storybook/aliases.ts

@@ -26,6 +26,7 @@ export const storybookAliases = {
   expression_shape: 'src/plugins/expression_shape/.storybook',
   expression_tagcloud: 'src/plugins/chart_expressions/expression_tagcloud/.storybook',
   expression_metric_vis: 'src/plugins/chart_expressions/expression_metric/.storybook',
+  expression_pie: 'src/plugins/chart_expressions/expression_pie/.storybook',
   fleet: 'x-pack/plugins/fleet/.storybook',
   infra: 'x-pack/plugins/infra/.storybook',
   security_solution: 'x-pack/plugins/security_solution/.storybook',

src/plugins/chart_expressions/expression_pie/.storybook/main.js

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// eslint-disable-next-line import/no-commonjs
+module.exports = require('@kbn/storybook').defaultConfig;

src/plugins/chart_expressions/expression_pie/common/expression_functions/pie_labels_function.ts

@@ -28,7 +28,7 @@ export const pieLabelsFunction = (): ExpressionFunctionDefinition<
       help: i18n.translate('expressionPie.pieLabels.function.args.show.help', {
         defaultMessage: 'Displays the pie labels',
       }),
-      required: true,
+      default: true,
     },
     position: {
       types: ['string'],