Merge branch 'main' into unified-field-list-add-new-fields

.buildkite/hooks/pre-command

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 
 if [[ "$BUILDKITE_AGENT_NAME" =~ ^bk-agent ]]; then
-  echo "Pipeline file triggered from outside the kibana executors, skipping pre_command"
+  echo "Pipeline file triggered from outside the kibana executors, skipping .buildkite/scripts/lifecycle/pre_command.sh"
+  export SONAR_LOGIN=$(vault read -field=token secret/ci/elastic-kibana/sonarqube)
 else
   source .buildkite/scripts/lifecycle/pre_command.sh
 fi

.buildkite/package.json

@@ -8,7 +8,7 @@
   },
   "dependencies": {
     "@octokit/rest": "^18.10.0",
-    "axios": "^0.21.4",
+    "axios": "^1.6.3",
     "globby": "^11.1.0",
     "js-yaml": "^4.1.0",
     "minimatch": "^5.0.1",
@@ -30,4 +30,4 @@
     "#pipeline-utils": "./pipeline-utils/index.ts",
     "#pipeline-utils/*": "./pipeline-utils/*"
   }
-}
+}

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -218,6 +218,12 @@ export async function pickTestGroupRunOrder() {
   if (Number.isNaN(FTR_CONFIGS_RETRY_COUNT)) {
     throw new Error(`invalid FTR_CONFIGS_RETRY_COUNT: ${process.env.FTR_CONFIGS_RETRY_COUNT}`);
   }
+  const JEST_CONFIGS_RETRY_COUNT = process.env.JEST_CONFIGS_RETRY_COUNT
+    ? parseInt(process.env.JEST_CONFIGS_RETRY_COUNT, 10)
+    : 1;
+  if (Number.isNaN(JEST_CONFIGS_RETRY_COUNT)) {
+    throw new Error(`invalid JEST_CONFIGS_RETRY_COUNT: ${process.env.JEST_CONFIGS_RETRY_COUNT}`);
+  }
 
   const FTR_CONFIGS_DEPS =
     process.env.FTR_CONFIGS_DEPS !== undefined
@@ -417,10 +423,10 @@ export async function pickTestGroupRunOrder() {
             },
             retry: {
               automatic: [
-                {
-                  exit_status: '-1',
-                  limit: 3,
-                },
+                { exit_status: '-1', limit: 3 },
+                ...(JEST_CONFIGS_RETRY_COUNT > 0
+                  ? [{ exit_status: '*', limit: JEST_CONFIGS_RETRY_COUNT }]
+                  : []),
               ],
             },
           }
@@ -437,10 +443,10 @@ export async function pickTestGroupRunOrder() {
             },
             retry: {
               automatic: [
-                {
-                  exit_status: '-1',
-                  limit: 3,
-                },
+                { exit_status: '-1', limit: 3 },
+                ...(JEST_CONFIGS_RETRY_COUNT > 0
+                  ? [{ exit_status: '*', limit: JEST_CONFIGS_RETRY_COUNT }]
+                  : []),
               ],
             },
           }

.buildkite/pipeline-utils/test-failures/annotate.ts

@@ -91,11 +91,12 @@ export const getPrComment = (
 
         const logsLink = artifactUrl ? ` [[logs]](${artifactUrl})` : '';
 
-        // job name could have #<number> in it, which Github will link to an issue, so we need to "escape" it with spans
-        return `* [[job]](${jobUrl})${logsLink} ${failure.jobName.replace(
-          '#',
-          '#<span></span>'
-        )} / ${failure.name}`;
+        // failure name could have #<number>, which Github will link to an issue or @<string>,
+        // which will send a notification so we need to "escape" it with spans
+        const failureString = `${failure.jobName} / ${failure.name}`
+          .replaceAll('#', '#<span></span>')
+          .replaceAll('@', '@<span></span>');
+        return `* [[job]](${jobUrl})${logsLink} ${failureString}`;
       })
       .join('\n')
   );

.buildkite/pipelines/artifacts.yml

@@ -51,16 +51,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: KIBANA_DOCKER_CONTEXT=ubi8 .buildkite/scripts/steps/artifacts/docker_context.sh
-    label: 'Docker Context Verification'
-    agents:
-      queue: n2-2
-    timeout_in_minutes: 30
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
   - command: KIBANA_DOCKER_CONTEXT=ubi .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:

.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml

@@ -11,6 +11,7 @@ steps:
         ENVIRONMENT: ${ENVIRONMENT}
         EC_ENV: qa
         EC_REGION: aws-eu-west-1
+        RETRY_TESTS_ON_FAIL: "true"
       message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)"
 
   # TODO: Uncomment this code when the integration is ready.

.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml

@@ -19,6 +19,7 @@ steps:
         ENVIRONMENT: ${ENVIRONMENT}
         EC_ENV: staging
         EC_REGION: aws-us-east-1
+        RETRY_TESTS_ON_FAIL: "true"
       message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
 
   - wait: ~

.buildkite/pipelines/sonarqube.yml

@@ -1,4 +1,5 @@
-
-steps:
-  - label: Placeholder
-    command: echo "Hello!"
+  - label: ":sonarqube: Continuous Code Inspection"
+    agents:
+      image: docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest
+      memory: 16G
+    command: /scan-source-code.sh

.buildkite/scripts/lifecycle/pre_command.sh

@@ -142,6 +142,12 @@ export SYNTHETICS_REMOTE_KIBANA_URL
 DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(vault_get kibana-serverless-release-tools DEPLOY_TAGGER_SLACK_WEBHOOK_URL)"}
 export DEPLOY_TAGGER_SLACK_WEBHOOK_URL
 
+GCS_SA_CDN_QA_KEY="$(vault_get gcs-sa-cdn-qa key)"
+export GCS_SA_CDN_QA_KEY
+
+GCS_SA_CDN_QA_BUCKET="$(vault_get gcs-sa-cdn-qa bucket)"
+export GCS_SA_CDN_QA_BUCKET
+
 # Setup Failed Test Reporter Elasticsearch credentials
 {
   TEST_FAILURES_ES_CLOUD_ID=$(vault_get failed_tests_reporter_es cloud_id)

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -23,11 +23,7 @@ case $KIBANA_DOCKER_CONTEXT in
   cloud)
     DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
   ;;
-  ubi8)
-    DOCKER_CONTEXT_FILE="kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
-  ;;
   ubi)
-    # Currently ubi9.  After ubi8 we're moving to a version agnostic filename
     DOCKER_CONTEXT_FILE="kibana-ubi-$FULL_VERSION-docker-build-context.tar.gz"
   ;;
   ironbank)