Merge branch 'master' into issues/113094

.bazelrc

@@ -2,12 +2,13 @@
 # Import shared settings first so we can override below
 import %workspace%/.bazelrc.common
 
-## Disabled for now
 # Remote cache settings for local env
-# build --remote_cache=https://storage.googleapis.com/kibana-bazel-cache
-# build --incompatible_remote_results_ignore_disk=true
-# build --remote_accept_cached=true
-# build --remote_upload_local_results=false
+build --remote_cache=grpcs://cloud.buildbuddy.io
+build --incompatible_remote_results_ignore_disk=true
+build --noremote_upload_local_results
+build --remote_timeout=30
+build --remote_header=x-buildbuddy-api-key=3EYk49W2NefOx2n3yMze
+build --remote_accept_cached=true
 
 # BuildBuddy
 ## Metadata settings

.buildkite/pipelines/hourly.yml

@@ -98,7 +98,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: ci-group-4d
+      queue: ci-group-6
     depends_on: build
     timeout_in_minutes: 120
     retry:

.buildkite/pipelines/pull_request/base.yml

@@ -96,7 +96,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: ci-group-4d
+      queue: ci-group-6
     depends_on: build
     timeout_in_minutes: 120
     retry:

.buildkite/pull_requests.json

@@ -0,0 +1,21 @@
+{
+  "jobs": [
+    {
+      "repoOwner": "elastic",
+      "repoName": "kibana",
+      "pipelineSlug": "kibana-pull-request",
+
+      "enabled": true,
+      "allow_org_users": true,
+      "allowed_repo_permissions": ["admin", "write"],
+      "allowed_list": ["barlowm", "renovate[bot]"],
+      "set_commit_status": true,
+      "commit_status_context": "kibana-ci",
+      "build_on_commit": true,
+      "build_on_comment": true,
+      "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
+      "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
+      "labels": ["buildkite-ci"]
+    }
+  ]
+}

.buildkite/scripts/build_kibana.sh

@@ -5,7 +5,11 @@ set -euo pipefail
 export KBN_NP_PLUGINS_BUILT=true
 
 echo "--- Build Kibana Distribution"
-node scripts/build --debug
+if [[ "${GITHUB_PR_LABELS:-}" == *"ci:build-all-platforms"* ]]; then
+  node scripts/build --all-platforms --skip-os-packages
+else
+  node scripts/build
+fi
 
 echo "--- Archive Kibana Distribution"
 linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"

.buildkite/scripts/build_kibana_plugins.sh

@@ -8,8 +8,6 @@ node scripts/build_kibana_platform_plugins \
   --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \
   --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
   --scan-dir "$KIBANA_DIR/examples" \
-  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
-  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
   --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
   --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
   --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \

.buildkite/scripts/common/env.sh

@@ -36,7 +36,12 @@ export ELASTIC_APM_ENVIRONMENT=ci
 export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1
 
 if is_pr; then
-  export ELASTIC_APM_ACTIVE=false
+  if [[ "${GITHUB_PR_LABELS:-}" == *"ci:collect-apm"* ]]; then
+    export ELASTIC_APM_ACTIVE=true
+  else
+    export ELASTIC_APM_ACTIVE=false
+  fi
+
   export CHECKS_REPORTER_ACTIVE=true
 
   # These can be removed once we're not supporting Jenkins and Buildkite at the same time

.buildkite/scripts/common/persist_bazel_cache.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common/util.sh
+
+KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
+export KIBANA_BUILDBUDDY_CI_API_KEY
+
+# overwrites the file checkout .bazelrc file with the one intended for CI env
+cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$KIBANA_DIR/.bazelrc"
+
+###
+### append auth token to buildbuddy into "$KIBANA_DIR/.bazelrc";
+###
+echo "# Appended by .buildkite/scripts/persist_bazel_cache.sh" >> "$KIBANA_DIR/.bazelrc"
+echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$KIBANA_DIR/.bazelrc"

.buildkite/scripts/lifecycle/post_command.sh

@@ -2,9 +2,12 @@
 
 set -euo pipefail
 
+node .buildkite/scripts/lifecycle/print_agent_links.js || true
+
 IS_TEST_EXECUTION_STEP="$(buildkite-agent meta-data get "${BUILDKITE_JOB_ID}_is_test_execution_step" --default '')"
 
 if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
+  echo "--- Upload Artifacts"
   buildkite-agent artifact upload 'target/junit/**/*'
   buildkite-agent artifact upload 'target/kibana-*'
   buildkite-agent artifact upload 'target/kibana-coverage/jest/**/*'
@@ -22,6 +25,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'x-pack/test/functional/failure_debug/html/*.html'
   buildkite-agent artifact upload '.es/**/*.hprof'
 
+  echo "--- Run Failed Test Reporter"
   node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml'
 
   if [[ -d 'target/test_failures' ]]; then

.buildkite/scripts/lifecycle/pre_command.sh

@@ -4,16 +4,17 @@ set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
 
-node .buildkite/scripts/lifecycle/print_agent_links.js
-
-echo '--- Job Environment Setup'
+BUILDKITE_TOKEN="$(retry 5 5 vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)"
+export BUILDKITE_TOKEN
 
+echo '--- Install buildkite dependencies'
 cd '.buildkite'
-yarn install
+retry 5 15 yarn install
 cd -
 
-BUILDKITE_TOKEN="$(retry 5 5 vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)"
-export BUILDKITE_TOKEN
+node .buildkite/scripts/lifecycle/print_agent_links.js || true
+
+echo '--- Job Environment Setup'
 
 # Set up a custom ES Snapshot Manifest if one has been specified for this build
 {
@@ -89,7 +90,6 @@ if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then
   if [[ -d .buildkite/scripts && "${BUILDKITE_COMMAND:-}" != "buildkite-agent pipeline upload"* ]]; then
     source .buildkite/scripts/common/env.sh
     source .buildkite/scripts/common/setup_node.sh
-    source .buildkite/scripts/common/setup_bazel.sh
   fi
 fi
 

.buildkite/scripts/lifecycle/print_agent_links.js

@@ -1,21 +1,38 @@
-const NOW = new Date();
-const TWO_HOURS = new Date(NOW.getTime() + 2 * 60 * 60 * 1000);
+const { BuildkiteClient } = require('kibana-buildkite-library');
 
-const METRICS_URL = [
-  `https://kibana-ops-buildkite-monitoring.kb.us-central1.gcp.cloud.es.io:9243`,
-  `/app/metrics/link-to/host-detail/${process.env.BUILDKITE_AGENT_NAME}`,
-  `?to=${TWO_HOURS.getTime()}`,
-  `&from=${NOW.getTime()}`,
-].join('');
+(async () => {
+  try {
+    const client = new BuildkiteClient();
+    const build = await client.getCurrentBuild();
 
-const LOGS_URL = [
-  `https://kibana-ops-buildkite-monitoring.kb.us-central1.gcp.cloud.es.io:9243`,
-  `/app/logs/link-to/host-logs/${process.env.BUILDKITE_AGENT_NAME}`,
-  `?time=${NOW.getTime()}`,
-].join('');
+    const job = build.jobs.find((j) => j.id === process.env.BUILDKITE_JOB_ID);
+    const startTime = job ? new Date(job.started_at) : new Date().getTime() - 60 * 60 * 1000;
+    const twoHours = new Date(startTime.getTime() + 2 * 60 * 60 * 1000);
 
-console.log('--- Agent Debug Links');
-console.log('Agent Metrics:');
-console.log('\u001b]1339;' + `url='${METRICS_URL}'\u0007`);
-console.log('Agent Logs:');
-console.log('\u001b]1339;' + `url='${LOGS_URL}'\u0007`);
+    const METRICS_URL = [
+      `https://kibana-ops-buildkite-monitoring.kb.us-central1.gcp.cloud.es.io:9243`,
+      `/app/metrics/link-to/host-detail/${process.env.BUILDKITE_AGENT_NAME}`,
+      `?to=${twoHours.getTime()}`,
+      `&from=${startTime.getTime()}`,
+    ].join('');
+
+    const LOGS_URL = [
+      `https://kibana-ops-buildkite-monitoring.kb.us-central1.gcp.cloud.es.io:9243`,
+      `/app/logs/link-to/host-logs/${process.env.BUILDKITE_AGENT_NAME}`,
+      `?time=${startTime.getTime()}`,
+    ].join('');
+
+    console.log('--- Agent Debug Links');
+    console.log('Agent Metrics:');
+    console.log('\u001b]1339;' + `url='${METRICS_URL}'\u0007`);
+    console.log('Agent Logs:');
+    console.log('\u001b]1339;' + `url='${LOGS_URL}'\u0007`);
+  } catch (ex) {
+    // Probably don't need to fail the build for this failure, just log it
+    console.error('Buildkite API Error', ex.message);
+    if (ex.response) {
+      console.error('HTTP Error Response Status', ex.response.status);
+      console.error('HTTP Error Response Body', ex.response.data);
+    }
+  }
+})();

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -55,21 +55,20 @@ const uploadPipeline = (pipelineContent) => {
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/base.yml', false));
 
     if (
-      await doAnyChangesMatch([
+      (await doAnyChangesMatch([
         /^x-pack\/plugins\/security_solution/,
         /^x-pack\/test\/security_solution_cypress/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/action_connector_form/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/context\/actions_connectors_context\.tsx/,
-      ])
+      ])) ||
+      process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/security_solution.yml'));
     }
 
-    // Disabled for now, these are failing/disabled in Jenkins currently as well
     // if (
-    //   await doAnyChangesMatch([
-    //     /^x-pack\/plugins\/apm/,
-    //   ])
+    //   (await doAnyChangesMatch([/^x-pack\/plugins\/apm/])) ||
+    //   process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     // ) {
     //   pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
     // }

.buildkite/scripts/post_build_kibana.sh

@@ -12,7 +12,6 @@ fi
 echo "--- Upload Build Artifacts"
 # Moving to `target/` first will keep `buildkite-agent` from including directories in the artifact name
 cd "$KIBANA_DIR/target"
-mv kibana-*-linux-x86_64.tar.gz kibana-default.tar.gz
-buildkite-agent artifact upload kibana-default.tar.gz
-buildkite-agent artifact upload kibana-default-plugins.tar.gz
+cp kibana-*-linux-x86_64.tar.gz kibana-default.tar.gz
+buildkite-agent artifact upload "./*.tar.gz;./*.zip"
 cd -

.buildkite/scripts/steps/es_snapshots/build.sh

@@ -61,7 +61,14 @@ export DOCKER_TLS_CERTDIR="$CERTS_DIR"
 export DOCKER_HOST=localhost:2377
 
 echo "--- Build Elasticsearch"
-./gradlew -Dbuild.docker=true assemble --parallel
+./gradlew \
+  :distribution:archives:darwin-aarch64-tar:assemble \
+  :distribution:archives:darwin-tar:assemble \
+  :distribution:docker:docker-export:assemble \
+  :distribution:archives:linux-aarch64-tar:assemble \
+  :distribution:archives:linux-tar:assemble \
+  :distribution:archives:windows-zip:assemble \
+  --parallel
 
 echo "--- Create distribution archives"
 find distribution -type f \( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \) -not -path '*no-jdk*' -not -path '*build-context*' -exec cp {} "$destination" \;

.buildkite/scripts/steps/functional/oss_misc.sh

@@ -2,9 +2,6 @@
 
 set -euo pipefail
 
-# Required, at least for kbn_sample_panel_action
-export BUILD_TS_REFS_DISABLE=false
-
 source .buildkite/scripts/steps/functional/common.sh
 
 # Required, at least for plugin_functional tests

.buildkite/scripts/steps/on_merge_build_and_metrics.sh

@@ -2,6 +2,9 @@
 
 set -euo pipefail
 
+# Write Bazel cache for Linux
+.buildkite/scripts/common/persist_bazel_cache.sh
+
 .buildkite/scripts/bootstrap.sh
 .buildkite/scripts/build_kibana.sh
 .buildkite/scripts/post_build_kibana.sh

.buildkite/scripts/steps/storybooks/build_and_upload.js

@@ -8,6 +8,7 @@ const STORYBOOKS = [
   'canvas',
   'codeeditor',
   'ci_composite',
+  'custom_integrations',
   'url_template_editor',
   'dashboard',
   'dashboard_enhanced',

.ci/.storybook/main.js

@@ -11,6 +11,12 @@ const aliases = require('../../src/dev/storybook/aliases.ts').storybookAliases;
 
 config.refs = {};
 
+// Required due to https://github.com/storybookjs/storybook/issues/13834
+config.babel = async (options) => ({
+  ...options,
+  plugins: ['@babel/plugin-transform-typescript', ...options.plugins],
+});
+
 for (const alias of Object.keys(aliases).filter((a) => a !== 'ci_composite')) {
   // snake_case -> Title Case
   const title = alias

.ci/Dockerfile

@@ -1,7 +1,7 @@
 # NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
 # If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
 
-ARG NODE_VERSION=14.17.6
+ARG NODE_VERSION=16.11.1
 
 FROM node:${NODE_VERSION} AS base
 
@@ -10,7 +10,7 @@ RUN apt-get update && \
       libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
       libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \
       libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
-      libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-8-jre && \
+      libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-11-jre && \
     rm -rf /var/lib/apt/lists/*
 
 RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \