Merge branch 'main' into monitoring-ts-convert-routes-elasticsearch

.buildkite/pipelines/es_snapshots/verify.yml

@@ -42,7 +42,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
-    parallelism: 11
+    parallelism: 12
     agents:
       queue: ci-group-4d
     depends_on: build

.buildkite/pipelines/on_merge.yml

@@ -66,7 +66,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
-    parallelism: 11
+    parallelism: 12
     agents:
       queue: n2-4-spot
     depends_on: build

.buildkite/pipelines/pull_request/base.yml

@@ -32,7 +32,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
-    parallelism: 11
+    parallelism: 12
     agents:
       queue: n2-4-spot
     depends_on: build

.ci/ci_groups.yml

@@ -10,6 +10,7 @@ root:
   - ciGroup9
   - ciGroup10
   - ciGroup11
+  - ciGroup12
 
 xpack:
   - ciGroup1

.github/CODEOWNERS

@@ -415,6 +415,7 @@
 /x-pack/plugins/security_solution/cypress/integration/urls @elastic/security-threat-hunting-investigations
 
 /x-pack/plugins/security_solution/public/common/components/alerts_viewer @elastic/security-threat-hunting-investigations
+/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_action @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/common/components/event_details @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/common/components/events_viewer @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/common/components/markdown_editor @elastic/security-threat-hunting-investigations

docs/management/connectors/action-types/email.asciidoc

@@ -9,6 +9,8 @@ The email connector uses the SMTP protocol to send mail messages, using an integ
 
 NOTE: For emails to have a footer with a link back to {kib}, set the <<server-publicBaseUrl, `server.publicBaseUrl`>> configuration setting.
 
+NOTE: When the <<action-config-email-domain-allowlist, `xpack.actions.email.domain_allowlist`>> configuration setting is used, the email addresses used for all of the Sender (from), To, CC, and BCC properties must have email domains specified in the configuration setting.
+
 [float]
 [[email-connector-configuration]]
 ==== Connector configuration

docs/settings/alert-action-settings.asciidoc

@@ -125,6 +125,11 @@ The contents of a PEM-encoded certificate file, or multiple files appended
 into a single string.  This configuration can be used for environments where
 the files cannot be made available.
 
+[[action-config-email-domain-allowlist]] `xpack.actions.email.domain_allowlist`  {ess-icon}::
+A list of allowed email domains which can be used with the email connector.  When this setting is not used, all email domains are allowed.  When this setting is used, if any email is attempted to be sent that includes an addressee with an email domain that is not in the allowlist, or the from address domain is not in the allowlist, the run of the connector will fail with a message indicating the emails not allowed.
+
+WARNING: This feature is available in {kib} 7.17.4 and 8.3.0 onwards but is not supported in {kib} 8.0, 8.1 or 8.2. As such this settings should be removed before upgrading from 7.17 to 8.0, 8.1 or 8.2. It is possible to configure the settings in 7.17.4 and then upgrade to 8.3.0 directly.
+
 `xpack.actions.enabledActionTypes` {ess-icon}::
 A list of action types that are enabled. It defaults to `[*]`, enabling all types. The names for built-in {kib} action types are prefixed with a `.` and include: `.email`, `.index`, `.jira`, `.pagerduty`, `.resilient`, `.server-log`, `.servicenow`, .`servicenow-itom`, `.servicenow-sir`, `.slack`, `.swimlane`, `.teams`, `.xmatters`, and `.webhook`. An empty list `[]` will disable all action types.
 +

package.json

@@ -250,6 +250,7 @@
     "deepmerge": "^4.2.2",
     "del": "^5.1.0",
     "elastic-apm-node": "^3.31.0",
+    "email-addresses": "^5.0.0",
     "execa": "^4.0.2",
     "exit-hook": "^2.2.0",
     "expiry-js": "0.1.7",

packages/kbn-optimizer/limits.yml

@@ -1,5 +1,6 @@
 pageLoadAssetSize:
   advancedSettings: 27596
+  actions: 20000
   alerting: 106936
   apm: 64385
   canvas: 1066647

packages/kbn-optimizer/src/babel_runtime_helpers/find_node_libs_browser_polyfills_in_entry_bundles.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+
+import { run } from '@kbn/dev-utils';
+import { REPO_ROOT } from '@kbn/utils';
+
+import { OptimizerConfig } from '../optimizer';
+import { parseStats, inAnyEntryChunk } from './parse_stats';
+
+export async function runFindNodeLibsBrowserPolyfillsInEntryBundlesCli() {
+  run(async ({ log }) => {
+    const config = OptimizerConfig.create({
+      includeCoreBundle: true,
+      repoRoot: REPO_ROOT,
+    });
+
+    const paths = config.bundles.map((b) => Path.resolve(b.outputDir, 'stats.json'));
+
+    log.info('analyzing', paths.length, 'stats files');
+    log.verbose(paths);
+
+    const imports = new Set();
+    for (const path of paths) {
+      const stats = parseStats(path);
+
+      for (const module of stats.modules) {
+        if (!inAnyEntryChunk(stats, module)) {
+          continue;
+        }
+
+        // Relying on module name instead of actual imports because these are usual polyfills that assume the global
+        // Node.js environment when development (i.e.: Buffer doesn't require an import to be used).
+        if (module.name.includes('node-libs-browser/node_modules/')) {
+          imports.add(module.name);
+        }
+      }
+    }
+
+    log.success('found', imports.size, 'node-libs-browser/* imports in entry bundles');
+    log.write(
+      Array.from(imports, (i) => `'${i}',`)
+        .sort()
+        .join('\n')
+    );
+  });
+}

packages/kbn-optimizer/src/babel_runtime_helpers/index.ts

@@ -7,3 +7,4 @@
  */
 
 export * from './find_babel_runtime_helpers_in_entry_bundles';
+export * from './find_node_libs_browser_polyfills_in_entry_bundles';

packages/kbn-optimizer/src/babel_runtime_helpers/parse_stats.ts

@@ -20,6 +20,7 @@ const partialObject = <P extends Props>(props: P) => {
 export type Module = TypeOf<typeof moduleSchema>;
 const moduleSchema = partialObject({
   identifier: schema.string(),
+  name: schema.string(),
   chunks: schema.arrayOf(schema.oneOf([schema.string(), schema.number()])),
   reasons: schema.arrayOf(
     partialObject({

packages/kbn-tooling-log/BUILD.bazel

@@ -54,6 +54,7 @@ TYPES_DEPS = [
   "@npm//tslib",
   "@npm//@types/node",
   "@npm//@types/jest",
+  "//packages/kbn-jest-serializers:npm_module_types" # needed for windows development, only used in tests
 ]
 
 jsts_transpiler(

packages/kbn-ui-shared-deps-npm/webpack.config.js

@@ -37,6 +37,10 @@ module.exports = (_, argv) => {
         'regenerator-runtime/runtime',
         'whatwg-fetch',
         'symbol-observable',
+        // Parts of node-libs-browser that are used in many places across Kibana
+        'buffer',
+        'punycode',
+        'util',
 
         /**
          * babel runtime helpers referenced from entry chunks

scripts/find_node_libs_browser_polyfills_in_use.js

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('../src/setup_node_env/no_transpilation');
+require('@kbn/optimizer').runFindNodeLibsBrowserPolyfillsInEntryBundlesCli();

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -188,6 +188,7 @@ kibana_vars=(
     vis_type_vega.enableExternalUrls
     xpack.actions.allowedHosts
     xpack.actions.customHostSettings
+    xpack.actions.email.domain_allowlist
     xpack.actions.enabledActionTypes
     xpack.actions.maxResponseContentLength
     xpack.actions.preconfigured

src/plugins/discover/public/components/discover_grid/discover_grid_cell_actions.test.tsx

@@ -48,7 +48,11 @@ describe('Discover cell actions ', function () {
     );
     const button = findTestSubject(component, 'filterForButton');
     await button.simulate('click');
-    expect(contextMock.onFilter).toHaveBeenCalledWith('extension', 'jpg', '+');
+    expect(contextMock.onFilter).toHaveBeenCalledWith(
+      indexPatternMock.fields.getByName('extension'),
+      'jpg',
+      '+'
+    );
   });
   it('triggers filter function when FilterOutBtn is clicked', async () => {
     const contextMock = {
@@ -76,6 +80,10 @@ describe('Discover cell actions ', function () {
     );
     const button = findTestSubject(component, 'filterOutButton');
     await button.simulate('click');
-    expect(contextMock.onFilter).toHaveBeenCalledWith('extension', 'jpg', '-');
+    expect(contextMock.onFilter).toHaveBeenCalledWith(
+      indexPatternMock.fields.getByName('extension'),
+      'jpg',
+      '-'
+    );
   });
 });

src/plugins/discover/public/components/discover_grid/discover_grid_cell_actions.tsx

@@ -11,7 +11,22 @@ import { EuiDataGridColumnCellActionProps } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { DataViewField } from '@kbn/data-views-plugin/public';
 import { flattenHit } from '@kbn/data-plugin/public';
-import { DiscoverGridContext } from './discover_grid_context';
+import { DiscoverGridContext, GridContext } from './discover_grid_context';
+
+function onFilterCell(
+  context: GridContext,
+  rowIndex: EuiDataGridColumnCellActionProps['rowIndex'],
+  columnId: EuiDataGridColumnCellActionProps['columnId'],
+  mode: '+' | '-'
+) {
+  const row = context.rows[rowIndex];
+  const flattened = flattenHit(row, context.indexPattern);
+  const field = context.indexPattern.fields.getByName(columnId);
+
+  if (flattened && field) {
+    context.onFilter(field, flattened[columnId], mode);
+  }
+}
 
 export const FilterInBtn = ({
   Component,
@@ -27,12 +42,7 @@ export const FilterInBtn = ({
   return (
     <Component
       onClick={() => {
-        const row = context.rows[rowIndex];
-        const flattened = flattenHit(row, context.indexPattern);
-
-        if (flattened) {
-          context.onFilter(columnId, flattened[columnId], '+');
-        }
+        onFilterCell(context, rowIndex, columnId, '+');
       }}
       iconType="plusInCircle"
       aria-label={buttonTitle}
@@ -60,12 +70,7 @@ export const FilterOutBtn = ({
   return (
     <Component
       onClick={() => {
-        const row = context.rows[rowIndex];
-        const flattened = flattenHit(row, context.indexPattern);
-
-        if (flattened) {
-          context.onFilter(columnId, flattened[columnId], '-');
-        }
+        onFilterCell(context, rowIndex, columnId, '-');
       }}
       iconType="minusInCircle"
       aria-label={buttonTitle}

src/plugins/vis_types/timelion/public/helpers/arg_value_suggestions.ts

@@ -24,7 +24,7 @@ export function getArgValueSuggestions() {
     }
     const indexPatternTitle = get(indexPatternArg, 'value.text');
 
-    return (await indexPatterns.find(indexPatternTitle)).find(
+    return (await indexPatterns.find(indexPatternTitle, 1)).find(
       (index) => index.title === indexPatternTitle
     );
   }

src/plugins/vis_types/timelion/public/timelion_vis_type.tsx

@@ -57,7 +57,7 @@ export function getTimelionVisDefinition(dependencies: TimelionVisDependencies)
         );
 
         if (indexArg?.value.text) {
-          return getIndexPatterns().find(indexArg.value.text);
+          return getIndexPatterns().find(indexArg.value.text, 1);
         }
       } catch {
         // timelion expression is invalid

src/plugins/vis_types/timelion/server/series_functions/es/index.js

@@ -104,7 +104,7 @@ export default new Datasource('es', {
       fit: 'nearest',
     });
     const indexPatternsService = tlConfig.getIndexPatternsService();
-    const indexPatternSpec = (await indexPatternsService.find(config.index)).find(
+    const indexPatternSpec = (await indexPatternsService.find(config.index, 1)).find(
       (index) => index.title === config.index
     );
 

test/functional/apps/visualize/index.ts

@@ -97,8 +97,13 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
     describe('visualize ciGroup11', function () {
       this.tags('ciGroup11');
 
-      loadTestFile(require.resolve('./_tag_cloud'));
       loadTestFile(require.resolve('./_tsvb_time_series'));
+    });
+
+    describe('visualize ciGroup12', function () {
+      this.tags('ciGroup12');
+
+      loadTestFile(require.resolve('./_tag_cloud'));
       loadTestFile(require.resolve('./_tsvb_markdown'));
       loadTestFile(require.resolve('./_tsvb_table'));
       loadTestFile(require.resolve('./_vega_chart'));

test/functional/page_objects/home_page.ts

@@ -93,6 +93,21 @@ export class HomePageObject extends FtrService {
     await this.find.clickByLinkText('Map');
   }
 
+  async launchSampleLogs(id: string) {
+    await this.launchSampleDataSet(id);
+    await this.find.clickByLinkText('Logs');
+  }
+
+  async launchSampleGraph(id: string) {
+    await this.launchSampleDataSet(id);
+    await this.find.clickByLinkText('Graph');
+  }
+
+  async launchSampleML(id: string) {
+    await this.launchSampleDataSet(id);
+    await this.find.clickByLinkText('ML jobs');
+  }
+
   async launchSampleDataSet(id: string) {
     await this.addSampleDataSet(id);
     await this.common.closeToastIfExists();

test/functional/services/combo_box.ts

@@ -157,7 +157,7 @@ export class ComboBoxService extends FtrService {
    * @param comboBoxElement element that wraps up EuiComboBox
    */
   private async waitForOptionsListLoading(comboBoxElement: WebElementWrapper): Promise<void> {
-    await comboBoxElement.waitForDeletedByCssSelector('.euiLoadingSpinner');
+    await comboBoxElement.waitForDeletedByCssSelector('.euiLoadingSpinner', 50);
   }
 
   /**
@@ -255,7 +255,9 @@ export class ComboBoxService extends FtrService {
    * @param comboBoxElement element that wraps up EuiComboBox
    */
   public async closeOptionsList(comboBoxElement: WebElementWrapper): Promise<void> {
-    const isOptionsListOpen = await this.testSubjects.exists('~comboBoxOptionsList');
+    const isOptionsListOpen = await this.testSubjects.exists('~comboBoxOptionsList', {
+      timeout: 50,
+    });
     if (isOptionsListOpen) {
       const input = await comboBoxElement.findByTagName('input');
       await input.pressKeys(this.browser.keys.ESCAPE);
@@ -268,7 +270,10 @@ export class ComboBoxService extends FtrService {
    * @param comboBoxElement element that wraps up EuiComboBox
    */
   public async openOptionsList(comboBoxElement: WebElementWrapper): Promise<void> {
-    const isOptionsListOpen = await this.testSubjects.exists('~comboBoxOptionsList');
+    const isOptionsListOpen = await this.testSubjects.exists('~comboBoxOptionsList', {
+      timeout: 50,
+    });
+
     if (!isOptionsListOpen) {
       await this.retry.try(async () => {
         const toggleBtn = await comboBoxElement.findByTestSubject('comboBoxInput');

test/functional/services/lib/web_element_wrapper/web_element_wrapper.ts

@@ -686,17 +686,23 @@ export class WebElementWrapper {
    * @param {string} className
    * @return {Promise<void>}
    */
-  public async waitForDeletedByCssSelector(selector: string): Promise<void> {
-    await this.driver.manage().setTimeouts({ implicit: 1000 });
-    await this.driver.wait(
-      async () => {
-        const found = await this._webElement.findElements(this.By.css(selector));
-        return found.length === 0;
-      },
-      this.timeout,
-      `The element with ${selector} selector was still present after ${this.timeout} sec.`
-    );
-    await this.driver.manage().setTimeouts({ implicit: this.timeout });
+  public async waitForDeletedByCssSelector(
+    selector: string,
+    implicitTimeout = 1000
+  ): Promise<void> {
+    try {
+      await this.driver.manage().setTimeouts({ implicit: implicitTimeout });
+      await this.driver.wait(
+        async () => {
+          const found = await this._webElement.findElements(this.By.css(selector));
+          return found.length === 0;
+        },
+        this.timeout,
+        `The element with ${selector} selector was still present after ${this.timeout} sec.`
+      );
+    } finally {
+      await this.driver.manage().setTimeouts({ implicit: this.timeout });
+    }
   }
 
   /**

test/plugin_functional/test_suites/core_plugins/rendering.ts

@@ -136,6 +136,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'usageCollection.uiCounters.debug (boolean)',
         'usageCollection.uiCounters.enabled (boolean)',
         'vis_type_vega.enableExternalUrls (boolean)',
+        'xpack.actions.email.domain_allowlist (array)',
         'xpack.apm.profilingEnabled (boolean)',
         'xpack.apm.serviceMapEnabled (boolean)',
         'xpack.apm.ui.enabled (boolean)',