Merge branch 'main' into event-log/hidden-again

.buildkite/pipelines/performance/nightly.yml → .buildkite/pipelines/performance/daily.yml

@@ -6,7 +6,7 @@ steps:
         key: "performance-test-iteration-count"
         hint: "How many times you want to run tests? "
         required: true
-    if: build.env('ITERATION_COUNT_ENV') == null
+    if: build.env('PERF_TEST_COUNT') == null
 
   - label: ":male-mechanic::skin-tone-2: Pre-Build"
     command: .buildkite/scripts/lifecycle/pre_build.sh
@@ -19,10 +19,10 @@ steps:
       queue: c2-16
     key: build
 
-  - label: ":muscle: Performance Tests"
-    command: .buildkite/scripts/steps/functional/performance.sh
+  - label: ":muscle: Performance Tests with Playwright config"
+    command: .buildkite/scripts/steps/functional/performance_playwright.sh
     agents:
-      queue: ci-group-6
+      queue: c2-16
     depends_on: build
 
   - wait: ~

.buildkite/scripts/steps/functional/performance.sh → .buildkite/scripts/steps/functional/performance_playwright.sh

@@ -2,20 +2,22 @@
 
 set -uo pipefail
 
-if [ -z "${ITERATION_COUNT_ENV+x}" ]; then
-  ITERATION_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
+if [ -z "${PERF_TEST_COUNT+x}" ]; then
+  TEST_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
 else
-  ITERATION_COUNT=$ITERATION_COUNT_ENV
+  TEST_COUNT=$PERF_TEST_COUNT
 fi
 
-tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${ITERATION_COUNT} times"
+tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${TEST_COUNT} times"
 
 cat << EOF | buildkite-agent pipeline upload
 steps:
-  - command: .buildkite/scripts/steps/functional/performance_sub.sh
-    parallelism: "$ITERATION_COUNT"
+  - command: .buildkite/scripts/steps/functional/performance_sub_playwright.sh
+    parallelism: "$TEST_COUNT"
     concurrency: 20
     concurrency_group: 'performance-test-group'
+    agents:
+      queue: c2-16
 EOF
 
 

.buildkite/scripts/steps/functional/performance_sub_playwright.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+echo --- Run Performance Tests with Playwright config
+
+node scripts/es snapshot&
+
+esPid=$!
+
+export TEST_PERFORMANCE_PHASE=WARMUP
+export TEST_ES_URL=http://elastic:changeme@localhost:9200
+export TEST_ES_DISABLE_STARTUP=true
+export ELASTIC_APM_ACTIVE=false
+
+sleep 120
+
+cd "$XPACK_DIR"
+
+# warmup round 1
+checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Phase: WARMUP)" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+    --config "test/performance/config.playwright.ts";
+
+export TEST_PERFORMANCE_PHASE=TEST
+export ELASTIC_APM_ACTIVE=true
+
+checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Phase: TEST)" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+    --config "test/performance/config.playwright.ts";
+
+kill "$esPid"

.github/CODEOWNERS

@@ -313,9 +313,11 @@
 /x-pack/plugins/encrypted_saved_objects/ @elastic/kibana-security
 /x-pack/plugins/security/ @elastic/kibana-security
 /x-pack/test/api_integration/apis/security/ @elastic/kibana-security
+/x-pack/test/api_integration/apis/spaces/ @elastic/kibana-security
 /x-pack/test/ui_capabilities/ @elastic/kibana-security
 /x-pack/test/encrypted_saved_objects_api_integration/ @elastic/kibana-security
 /x-pack/test/functional/apps/security/ @elastic/kibana-security
+/x-pack/test/functional/apps/spaces/ @elastic/kibana-security
 /x-pack/test/security_api_integration/ @elastic/kibana-security
 /x-pack/test/security_functional/ @elastic/kibana-security
 /x-pack/test/spaces_api_integration/ @elastic/kibana-security

api_docs/cases.mdx

@@ -12,13 +12,13 @@ import casesObj from './cases.json';
 
 The Case management system in Kibana
 
-Contact [Security Solution Threat Hunting](https://github.com/orgs/elastic/teams/security-threat-hunting) for questions regarding this plugin.
+Contact [ResponseOps](https://github.com/orgs/elastic/teams/response-ops) for questions regarding this plugin.
 
 **Code health stats**
 
-| Public API count  | Any count | Items lacking comments | Missing exports |
-|-------------------|-----------|------------------------|-----------------|
-| 83 | 0 | 57 | 23 |
+| Public API count | Any count | Items lacking comments | Missing exports |
+| ---------------- | --------- | ---------------------- | --------------- |
+| 83               | 0         | 57                     | 23              |
 
 ## Client
 

api_docs/plugin_directory.mdx

@@ -31,7 +31,7 @@ warning: This document is auto-generated and is meant to be viewed inside our ex
 | <DocLink id="kibBannersPluginApi" text="banners"/> | [Kibana Core](https://github.com/orgs/elastic/teams/kibana-core) | - | 9 | 0 | 9 | 0 |
 | <DocLink id="kibBfetchPluginApi" text="bfetch"/> | [App Services](https://github.com/orgs/elastic/teams/kibana-app-services) | Considering using bfetch capabilities when fetching large amounts of data. This services supports batching HTTP requests and streaming responses back. | 76 | 1 | 67 | 2 |
 | <DocLink id="kibCanvasPluginApi" text="canvas"/> | [Kibana Presentation](https://github.com/orgs/elastic/teams/kibana-presentation) | Adds Canvas application to Kibana | 9 | 0 | 8 | 3 |
-| <DocLink id="kibCasesPluginApi" text="cases"/> | [Security Solution Threat Hunting](https://github.com/orgs/elastic/teams/security-threat-hunting) | The Case management system in Kibana | 83 | 0 | 57 | 23 |
+| <DocLink id="kibCasesPluginApi" text="cases"/> | [ResponseOps](https://github.com/orgs/elastic/teams/response-ops) | The Case management system in Kibana | 83 | 0 | 57 | 23 |
 | <DocLink id="kibChartsPluginApi" text="charts"/> | [Vis Editors](https://github.com/orgs/elastic/teams/kibana-vis-editors) | - | 314 | 2 | 281 | 4 |
 | <DocLink id="kibCloudPluginApi" text="cloud"/> | [Kibana Core](https://github.com/orgs/elastic/teams/kibana-core) | - | 22 | 0 | 22 | 0 |
 | <DocLink id="kibConsolePluginApi" text="console"/> | [Stack Management](https://github.com/orgs/elastic/teams/kibana-stack-management) | - | 13 | 0 | 13 | 1 |

docs/CHANGELOG.asciidoc

@@ -134,17 +134,19 @@ The `xpack.apm.autocreateApmIndexPattern` APM setting has been removed. For more
 *Impact* +
 To automatically create data views in APM, use `xpack.apm.autoCreateApmDataView`.
 ====
-      
+
 [discrete]
 [[deprecation-119494]]
-.Updates Fleet API responses for consistency
+.Updates Fleet API to improve consistency
 [%collapsible]
 ====
 *Details* +
-To make sure all Fleet API GET resposes return `items`, the following have been updated:
+The Fleet API has been updated to improve consistency:
 
-* `/api/fleet/enrollment-api-keys`
-* `/api/fleet/agents`
+* Hyphens are changed to underscores in some names.
+* The `pkgkey` path parameter in the packages endpoint is split.
+* The `response` and `list` properties are renamed to `items` or `item` in some
+responses.
 
 For more information, refer to {kibana-pull}119494[#119494].
 
@@ -157,24 +159,30 @@ When you upgrade to 8.0.0, use the following API changes:
 
 * Use `service_tokens` instead of `service-tokens`.
 
-* `check-permissions` is no longer supported.
-
 * Use `/epm/packages/{packageName}/{version}` instead of `/epm/packages/{pkgkey}`.
 
-* Use `items[]` or `item` instead of `response[]` in the following:
-
+* Use `items[]` instead of `response[]` in:
++
 [source,text]
 --
+/api/fleet/enrollment_api_keys
+/api/fleet/agents
 /epm/packages/
-/epm/packages/{pkgkey}
 /epm/categories
 /epm/packages/_bulk
 /epm/packages/limited
+/epm/packages/{packageName}/{version} <1>
 --
+<1> Use `items[]` when the verb is `POST` or `DELETE`. Use `item` when the verb
+is `GET` or `PUT`.
+
+For more information, refer to {fleet-guide}/fleet-api-docs.html[Fleet APIs].
+
 ====
 
-To review the depcrecations in previous versions, refer to the <<deprecations-8.0.0-alpha1,8.0.0-alpha1 release notes>>. 
-
+To review the deprecations in previous versions, refer to the <<deprecations-8.0.0-alpha1,8.0.0-alpha1 release notes>>. 
+
+
 [float]
 [[features-8.0.0-rc1]]
 === Features

docs/apm/troubleshooting.asciidoc

@@ -185,7 +185,7 @@ There are two things you can do to if you'd like to ensure a field is searchable
 1. Index your additional data as {apm-guide-ref}/metadata.html[labels] instead.
 These are dynamic by default, which means they will be indexed and become searchable and aggregatable.
 
-2. Use the {apm-guide-ref}/configuration-template.html[`append_fields`] feature. As an example,
+2. Use the `append_fields` feature. As an example,
 adding the following to `apm-server.yml` will enable dynamic indexing for `http.request.cookies`:
 
 [source,yml]

docs/management/connectors/action-types/email.asciidoc

@@ -16,7 +16,7 @@ NOTE: For emails to have a footer with a link back to {kib}, set the <<server-pu
 Email connectors have the following configuration properties.
 
 Name::      The name of the connector. The name is used to identify a  connector in the management UI connector listing, or in the connector list when configuring an action.
-Sender::    The from address for all emails sent with this connector. This can be specified in `user@host-name` format or as `"human name <user@host-name>"` format. See the https://nodemailer.com/message/addresses/[Nodemailer address documentation] for more information.
+Sender::    The from address for all emails sent with this connector. This must be specified in `user@host-name` format. See the https://nodemailer.com/message/addresses/[Nodemailer address documentation] for more information.
 Service::   The name of the email service. If `service` is one of Nodemailer's https://nodemailer.com/smtp/well-known/[well-known email service providers], the `host`, `port`, and `secure` properties are defined with the default values and disabled for modification. If `service` is `MS Exchange Server`, the `host`, `port`, and `secure` properties are ignored and `tenantId`, `clientId`, `clientSecret` are required instead. If `service` is `other`, the `host` and `port` properties must be defined.
 Host::      Host name of the service provider. If you are using the <<action-settings, `xpack.actions.allowedHosts`>> setting, make sure this hostname is added to the allowed hosts.
 Port::      The port to connect to on the service provider.

docs/osquery/osquery.asciidoc

@@ -121,11 +121,18 @@ image::images/scheduled-pack.png[Shows queries in the pack and details about eac
 
 [float]
 [[osquery-manage-query]]
-== Edit saved queries
+== Save queries
 
-Add or edit saved queries from the *Saved queries* tab.
+You can save queries in two ways:
 
-. Go to the saved queries, then click **Add saved query** or the edit icon.
+* After running a live query, click the *Save for later* link.
+* From the *Saved queries* tab, click the **Add saved query** button.
+
+Once you save a query, you can only edit it from the *Saved queries* tab.
+
+To add or edit saved queries from the *Saved queries* tab:
+
+. Go to *Saved queries*, and then click **Add saved query** or the edit icon.
 . Provide the following fields:
 
 * The unique identifier.
@@ -148,7 +155,7 @@ Add or edit saved queries from the *Saved queries* tab.
 
 * From the *Test query* panel, select agents or groups to test the query, then click *Submit* to run a live query. Result columns with the image:images/mapped-icon.png[mapping] icon are mapped. Hover over the icon to see the mapped ECS field.
 
-. Click **Save query**.
+. Click *Save* or *Update*. 
 
 [float]
 [[osquery-map-fields]]
@@ -175,11 +182,7 @@ and the mapped ECS fields. For example, if you update a query to map `osquery.na
 
 ** **Static value**: Enter a static value. When the query runs, the ECS field is set to the value entered. For example, static fields can be used to apply `tags` or your preferred `event.category` to the query results. 
 
-. Map more fields, as needed.
-
-** To add a new row for additional fields to map, click the plus icon.
-
-** To remove any mapped rows, click the trash icon.
+. Map more fields, as needed. To remove any mapped rows, click the delete icon.
 
 . Save your changes.
 
@@ -314,7 +317,7 @@ While this allows you to use advanced Osquery functionality like pack discovery
 
 . Edit the *Osquery config* JSON field to apply your preferred Osquery configuration. Note the following:
 
-* The field may already have content if you have scheduled packs for this agent policy. To keep these packs scheduled, do not edit the `packs` section.
+* The field may already have content if you have scheduled packs for this agent policy. To keep these packs scheduled, do not remove the `packs` section.
 
 * Refer to the https://osquery.readthedocs.io/en/stable/[Osquery documentation] for configuration options. 
 
@@ -344,14 +347,12 @@ https://www.elastic.co/guide/en/fleet/master/upgrade-elastic-agent.html[upgrade
 
 [float]
 === Debug issues
-If you encounter issues with *Osquery Manager*, find the relevant logs for the {elastic-agent}
-and Osquerybeat in the installed agent directory, then adjust the agent path for your setup. 
-
-The relevant logs look similar to the following example paths:
+If you encounter issues with *Osquery Manager*, find the relevant logs for {elastic-agent}
+and Osquerybeat in the agent directory. Refer to the {fleet-guide}/installation-layout.html[Fleet Installation layout] to find the log file location for your OS.
 
 ```ts
-`/data/elastic-agent-054e22/logs/elastic-agent-json.log-*`
-`/data/elastic-agent-054e22/logs/default/osquerybeat-json.log`
+../data/elastic-agent-*/logs/elastic-agent-json.log-*
+../data/elastic-agent-*/logs/default/osquerybeat-json.log
 ```
 
 To get more details in the logs, change the agent logging level to debug:

docs/settings/alert-action-settings.asciidoc

@@ -190,7 +190,7 @@ Specifies the default timeout for the all rule types tasks. The time is formatte
 +
 `<count>[ms,s,m,h,d,w,M,Y]` 
 +
-For example, `20m`, `24h`, `7d`, `1w`. Default: `60s`.
+For example, `20m`, `24h`, `7d`, `1w`. Default: `5m`.
 
 `xpack.alerting.cancelAlertsOnRuleTimeout`::
 Specifies whether to skip writing alerts and scheduling actions if rule execution is cancelled due to timeout. Default: `true`. This setting can be overridden by individual rule types.

package.json

@@ -725,7 +725,7 @@
     "cpy": "^8.1.1",
     "css-loader": "^3.4.2",
     "cssnano": "^4.1.11",
-    "cypress": "^9.2.0",
+    "cypress": "^9.2.1",
     "cypress-axe": "^0.14.0",
     "cypress-cucumber-preprocessor": "^2.5.2",
     "cypress-file-upload": "^5.0.8",
@@ -831,6 +831,7 @@
     "pbf": "3.2.1",
     "pirates": "^4.0.1",
     "pixelmatch": "^5.1.0",
+    "playwright": "^1.17.1",
     "postcss": "^7.0.32",
     "postcss-loader": "^3.0.0",
     "postcss-prefix-selector": "^1.7.2",

packages/elastic-apm-synthtrace/src/scripts/examples/01_simple_trace.ts

@@ -12,14 +12,14 @@ import { getApmWriteTargets } from '../../lib/apm/utils/get_apm_write_targets';
 import { Scenario } from '../scenario';
 import { getCommonServices } from '../utils/get_common_services';
 
-const scenario: Scenario = async ({ target, logLevel }) => {
+const scenario: Scenario = async ({ target, logLevel, scenarioOpts }) => {
   const { client, logger } = getCommonServices({ target, logLevel });
   const writeTargets = await getApmWriteTargets({ client });
 
+  const { numServices = 3 } = scenarioOpts || {};
+
   return {
     generate: ({ from, to }) => {
-      const numServices = 3;
-
       const range = timerange(from, to);
 
       const transactionName = '240rpm/75% 1000ms';

packages/elastic-apm-synthtrace/src/scripts/run.ts

@@ -69,6 +69,12 @@ function options(y: Argv) {
       describe: 'Target to index',
       string: true,
     })
+    .option('scenarioOpts', {
+      describe: 'Options specific to the scenario',
+      coerce: (arg) => {
+        return arg as Record<string, any> | undefined;
+      },
+    })
     .conflicts('to', 'live');
 }
 

packages/elastic-apm-synthtrace/src/scripts/utils/parse_run_cli_flags.ts

@@ -47,7 +47,15 @@ export function parseRunCliFlags(flags: RunCliFlags) {
   }
 
   return {
-    ...pick(flags, 'target', 'workers', 'clientWorkers', 'batchSize', 'writeTarget'),
+    ...pick(
+      flags,
+      'target',
+      'workers',
+      'clientWorkers',
+      'batchSize',
+      'writeTarget',
+      'scenarioOpts'
+    ),
     intervalInMs,
     bucketSizeInMs,
     logLevel: parsedLogLevel,

packages/elastic-apm-synthtrace/src/scripts/utils/start_historical_data_upload.ts

@@ -24,6 +24,7 @@ export async function startHistoricalDataUpload({
   target,
   file,
   writeTarget,
+  scenarioOpts,
 }: RunOptions & { from: number; to: number }) {
   let requestedUntil: number = from;
 
@@ -57,6 +58,7 @@ export async function startHistoricalDataUpload({
       target,
       workers,
       writeTarget,
+      scenarioOpts,
     };
 
     const worker = new Worker(Path.join(__dirname, './upload_next_batch.js'), {