Merge branch 'main' into serverless-js-doc

.buildkite/ftr_configs.yml

@@ -15,7 +15,7 @@ disabled:
   - x-pack/test/fleet_api_integration/config.base.ts
   - x-pack/test/security_solution_api_integration/config/ess/config.base.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
-
+  - x-pack/test/security_solution_endpoint/config.base.ts
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -387,7 +387,9 @@ enabled:
   - x-pack/test/security_functional/expired_session.config.ts
   - x-pack/test/security_solution_endpoint_api_int/config.ts
   - x-pack/test/security_solution_endpoint/endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/integrations.config.ts
+  - x-pack/test/security_solution_endpoint/serverless.integrations.config.ts
   - x-pack/test/session_view/basic/config.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts
@@ -457,7 +459,9 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts
-
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/ess.config.ts
+
 
 
 

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -273,7 +273,12 @@ export async function pickTestGroupRunOrder() {
           ]
         : []),
       // if we are running on a external job, like kibana-code-coverage-main, try finding times that are specific to that job
-      ...(!prNumber && pipelineSlug !== 'kibana-on-merge'
+      // kibana-elasticsearch-serverless-verify-and-promote is not necessarily run in commit order -
+      // using kibana-on-merge groups will provide a closer approximation, with a failure mode -
+      // of too many ftr groups instead of potential timeouts.
+      ...(!prNumber &&
+      pipelineSlug !== 'kibana-on-merge' &&
+      pipelineSlug !== 'kibana-elasticsearch-serverless-verify-and-promote'
         ? [
             {
               branch: ownBranch,

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -63,7 +63,7 @@ steps:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 4
+        parallelism: 6
         retry:
           automatic:
             - exit_status: '*'
@@ -102,7 +102,7 @@ steps:
           queue: n2-4-virt
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 4
+        parallelism: 6
         retry:
           automatic:
             - exit_status: '*'

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -162,10 +162,11 @@ for (const testSuite of testSuites) {
           `Group configuration was not found in groups.json for the following cypress suite: {${suiteName}}.`
         );
       }
+      const agentQueue = suiteName.includes('defend_workflows') ? 'n2-4-virt' : 'n2-4-spot';
       steps.push({
         command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
         label: group.name,
-        agents: { queue: 'n2-4-spot' },
+        agents: { queue: agentQueue },
         depends_on: 'build',
         parallelism: testSuite.count,
         concurrency,

.buildkite/pipelines/on_merge.yml

@@ -157,7 +157,7 @@ steps:
       queue: n2-4-virt
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 8
+    parallelism: 10
     retry:
       automatic:
         - exit_status: '*'
@@ -169,7 +169,7 @@ steps:
       queue: n2-4-virt
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 4
+    parallelism: 6
     retry:
       automatic:
         - exit_status: '*'
@@ -254,7 +254,7 @@ steps:
   - command: .buildkite/scripts/steps/check_types.sh
     label: 'Check Types'
     agents:
-      queue: n2-16-spot
+      queue: n2-4-spot
     timeout_in_minutes: 60
     retry:
       automatic:

.buildkite/pipelines/pull_request/base.yml

@@ -20,7 +20,7 @@ steps:
           limit: 3
 
   - command: .buildkite/scripts/steps/build_api_docs.sh
-    label: 'Build API Docs'
+    label: 'Check Types and Build API Docs'
     agents:
       queue: n2-4-spot
     key: build_api_docs
@@ -135,7 +135,7 @@ steps:
       queue: n2-4-virt
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 8
+    parallelism: 10
     retry:
       automatic:
         - exit_status: '*'
@@ -147,7 +147,7 @@ steps:
       queue: n2-4-virt
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 4
+    parallelism: 6
     retry:
       automatic:
         - exit_status: '*'
@@ -232,13 +232,3 @@ steps:
       automatic:
         - exit_status: '-1'
           limit: 3
-
-  - command: .buildkite/scripts/steps/check_types.sh
-    label: 'Check Types'
-    agents:
-      queue: n2-16-spot
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3

.buildkite/pipelines/pull_request/deploy_project.yml

@@ -0,0 +1,11 @@
+steps:
+  - command: .buildkite/scripts/steps/serverless/build_and_deploy.sh
+    label: 'Build and Deploy Project'
+    agents:
+      queue: n2-16-spot
+    timeout_in_minutes: 60
+    soft_fail: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml

@@ -39,8 +39,21 @@ steps:
 
   - wait: ~
 
-  - label: ":judge::seedling: Trigger Manual Tests Phase"
-    command: "make -C /agent trigger-manual-verification-phase"
-    if: build.branch == "main"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"
+  - group: "Kibana Release Manager"
+    steps:
+      - label: ":judge::seedling: Trigger Manual Tests Phase - Kibana Release Manager"
+        command: "make -C /agent trigger-manual-verification-phase"
+        env:
+          USE_GROUP_LABEL: true
+        agents:
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"
+
+  - group: "Observability"
+    steps:
+      - label: ":judge::seedling: Trigger Manual Tests Phase - Observability"
+        command: "make -C /agent trigger-manual-verification-phase"
+        env:
+          NOTIFICATION_APPENDIX: "<!subteam^S060GJFKGGP> please execute your manual testing plan."
+          USE_GROUP_LABEL: true
+        agents:
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"

.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml

@@ -23,8 +23,9 @@ steps:
 
   - wait: ~
 
-  - label: ":judge::seedling: Trigger Manual Tests Phase"
-    command: "make -C /agent trigger-manual-verification-phase"
-    if: build.branch == "main"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"
+  - group: "Kibana Release Manager"
+    steps:
+      - label: ":judge::seedling: Trigger Manual Tests Phase"
+        command: "make -C /agent trigger-manual-verification-phase"
+        agents:
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"

.buildkite/pipelines/security_solution/base.yml

@@ -0,0 +1,36 @@
+steps:
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless
+    label: 'Serverless MKI QA Security Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore
+    label: 'Serverless MKI QA Explore - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations
+    label: 'Serverless MKI QA Investigations - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/build_kibana.sh

@@ -14,6 +14,7 @@ is_pr_with_label "ci:build-docker-cross-compile" && BUILD_ARGS+=("--docker-cross
 is_pr_with_label "ci:build-os-packages" || BUILD_ARGS+=("--skip-os-packages")
 is_pr_with_label "ci:build-canvas-shareable-runtime" || BUILD_ARGS+=("--skip-canvas-shareable-runtime")
 is_pr_with_label "ci:build-docker-contexts" || BUILD_ARGS+=("--skip-docker-contexts")
+is_pr_with_label "ci:build-cdn-assets" || BUILD_ARGS+=("--skip-cdn-assets")
 
 echo "> node scripts/build" "${BUILD_ARGS[@]}"
 node scripts/build "${BUILD_ARGS[@]}"
@@ -24,6 +25,7 @@ if is_pr_with_label "ci:build-cloud-image"; then
   --skip-initialize \
   --skip-generic-folders \
   --skip-platform-folders \
+  --skip-cdn-assets \
   --skip-archives \
   --docker-images \
   --docker-tag-qualifier="$GIT_COMMIT" \

.buildkite/scripts/lifecycle/post_command.sh

@@ -14,7 +14,9 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*'
   buildkite-agent artifact upload 'target/kibana-*'
   buildkite-agent artifact upload 'target/kibana-security-solution/**/*.png'
+  buildkite-agent artifact upload 'target/kibana-security-solution/**/management/**/*.mp4'
   buildkite-agent artifact upload 'target/kibana-osquery/**/*.png'
+  buildkite-agent artifact upload 'target/kibana-osquery/**/*.mp4'
   buildkite-agent artifact upload 'target/kibana-fleet/**/*.png'
   buildkite-agent artifact upload 'target/test-metrics/*'
   buildkite-agent artifact upload 'target/test-suites-ci-plan.json'

.buildkite/scripts/lifecycle/pre_command.sh

@@ -115,6 +115,12 @@ export KIBANA_DOCKER_PASSWORD
 EC_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-cloud-deploy)"
 export EC_API_KEY
 
+PROJECT_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-project-deploy)"
+export PROJECT_API_KEY
+
+PROJECT_API_DOMAIN="$(retry 5 5 vault read -field=pr_deploy_domain secret/kibana-issues/dev/kibana-ci-project-deploy)"
+export PROJECT_API_DOMAIN
+
 SYNTHETICS_SERVICE_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
 export SYNTHETICS_SERVICE_USERNAME
 

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -136,6 +136,14 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_cloud.yml'));
     }
 
+    if (
+      GITHUB_PR_LABELS.includes('ci:project-deploy-es') ||
+      GITHUB_PR_LABELS.includes('ci:project-deploy-oblt') ||
+      GITHUB_PR_LABELS.includes('ci:project-deploy-security')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_project.yml'));
+    }
+
     if (GITHUB_PR_LABELS.includes('ci:build-serverless-image')) {
       pipeline.push(getPipeline('.buildkite/pipelines/artifacts_container_image.yml'));
     }

.buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [ -z "$1" ]
+  then
+    echo "No target script from the package.json file, is supplied"
+    exit 1
+fi
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+.buildkite/scripts/bootstrap.sh
+
+export JOB=kibana-security-solution-chrome
+
+buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
+
+cd x-pack/test/security_solution_cypress
+set +e
+
+QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key)
+
+CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh

@@ -1,19 +1,5 @@
 #!/bin/bash
-set -euo pipefail
-
-source .buildkite/scripts/common/util.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
-.buildkite/scripts/bootstrap.sh
-
-export JOB=kibana-security-solution-chrome
 
-buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
-
-echo "--- Serverless Security Second Quality Gate"
-cd x-pack/test/security_solution_cypress
-set +e
-
-VAULT_DEC_KEY=$(vault read -field=key secret/kibana-issues/dev/security-solution-qg-enc-key)
-ENV_PWD=$(echo $TEST_ENV_PWD | openssl aes-256-cbc -d -a -pass pass:$VAULT_DEC_KEY)
+set -euo pipefail
 
-CYPRESS_ELASTICSEARCH_URL=$TEST_ENV_ES_URL CYPRESS_BASE_URL=$TEST_ENV_KB_URL CYPRESS_ELASTICSEARCH_USERNAME=$TEST_ENV_USERNAME CYPRESS_ELASTICSEARCH_PASSWORD=$ENV_PWD CYPRESS_KIBANA_URL=$CYPRESS_BASE_URL yarn cypress:run:qa:serverless; status=$?; yarn junit:merge || :; exit $status
+ts-node .buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts

.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { execSync } from 'child_process';
+import fs from 'fs';
+
+const getPipeline = (filename: string, removeSteps = true) => {
+  const str = fs.readFileSync(filename).toString();
+  return removeSteps ? str.replace(/^steps:/, '') : str;
+};
+
+const uploadPipeline = (pipelineContent: string | object) => {
+  const str =
+    typeof pipelineContent === 'string' ? pipelineContent : JSON.stringify(pipelineContent);
+
+  execSync('buildkite-agent pipeline upload', {
+    input: str,
+    stdio: ['pipe', 'inherit', 'inherit'],
+  });
+};
+
+(async () => {
+  try {
+    const pipeline = [];
+
+    pipeline.push(getPipeline('.buildkite/pipelines/security_solution/base.yml', false));
+    // remove duplicated steps
+    uploadPipeline([...new Set(pipeline)].join('\n'));
+  } catch (ex) {
+    console.error('PR pipeline generation error', ex.message);
+    process.exit(1);
+  }
+})();

.buildkite/scripts/steps/archive_so_migration_snapshot.sh

@@ -7,7 +7,7 @@ SO_MIGRATIONS_SNAPSHOT_FOLDER=kibana-so-types-snapshots
 SNAPSHOT_FILE_PATH="${1:-target/plugin_so_types_snapshot.json}"
 
 echo "--- Creating snapshot of Saved Object migration info"
-node scripts/snapshot_plugin_types --outputPath "$SNAPSHOT_FILE_PATH"
+node scripts/snapshot_plugin_types snapshot --outputPath "$SNAPSHOT_FILE_PATH"
 
 echo "--- Uploading as ${BUILDKITE_COMMIT}.json"
 SNAPSHOT_PATH="${SO_MIGRATIONS_SNAPSHOT_FOLDER}/${BUILDKITE_COMMIT}.json"