Merge branch '7.x' into backport_25074_7.x

.ci/packaging.groovy

@@ -17,6 +17,7 @@ pipeline {
     JOB_GCS_BUCKET = 'beats-ci-artifacts'
     JOB_GCS_BUCKET_STASH = 'beats-ci-temp'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
+    JOB_GCS_EXT_CREDENTIALS = 'beats-ci-gcs-plugin-file-credentials'
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
     GITHUB_CHECK_E2E_TESTS_NAME = 'E2E Tests'
@@ -114,6 +115,7 @@ pipeline {
                    'x-pack/heartbeat',
                   // 'x-pack/journalbeat',
                   'x-pack/metricbeat',
+                  'x-pack/osquerybeat',
                   'x-pack/packetbeat',
                   'x-pack/winlogbeat'
                 )
@@ -151,7 +153,9 @@ pipeline {
                   withGithubNotify(context: "Packaging Linux ${BEATS_FOLDER}") {
                     deleteDir()
                     release()
-                    pushCIDockerImages()
+                    dir("${BASE_DIR}"){
+                      pushCIDockerImages(arch: 'amd64')
+                    }
                   }
                   prepareE2ETestForPackage("${BEATS_FOLDER}")
                 }
@@ -203,6 +207,7 @@ pipeline {
                   'metricbeat',
                   'packetbeat',
                   'x-pack/auditbeat',
+                  'x-pack/dockerlogbeat',
                   'x-pack/elastic-agent',
                   'x-pack/filebeat',
                   'x-pack/heartbeat',
@@ -232,7 +237,9 @@ pipeline {
                   withGithubNotify(context: "Packaging linux/arm64 ${BEATS_FOLDER}") {
                     deleteWorkspace()
                     release()
-                    pushCIDockerImages()
+                    dir("${BASE_DIR}"){
+                      pushCIDockerImages(arch: 'arm64')
+                    }
                   }
                 }
                 post {
@@ -268,27 +275,39 @@ pipeline {
   }
 }
 
-def pushCIDockerImages(){
+/**
+* @param arch what architecture
+*/
+def pushCIDockerImages(Map args = [:]) {
+  def arch = args.get('arch', 'amd64')
   catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
     if (env?.BEATS_FOLDER?.endsWith('auditbeat')) {
-      tagAndPush('auditbeat')
+      tagAndPush(beatName: 'auditbeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('filebeat')) {
-      tagAndPush('filebeat')
+      tagAndPush(beatName: 'filebeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('heartbeat')) {
-      tagAndPush('heartbeat')
+      tagAndPush(beatName: 'heartbeat', arch: arch)
     } else if ("${env.BEATS_FOLDER}" == "journalbeat"){
-      tagAndPush('journalbeat')
+      tagAndPush(beatName: 'journalbeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) {
-      tagAndPush('metricbeat')
+      tagAndPush(beatName: 'metricbeat', arch: arch)
+    } else if (env?.BEATS_FOLDER?.endsWith('osquerybeat')) {
+      tagAndPush(beatName: 'osquerybeat', arch: arch)
     } else if ("${env.BEATS_FOLDER}" == "packetbeat"){
-      tagAndPush('packetbeat')
+      tagAndPush(beatName: 'packetbeat', arch: arch)
     } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
-      tagAndPush('elastic-agent')
+      tagAndPush(beatName: 'elastic-agent', arch: arch)
     }
   }
 }
 
-def tagAndPush(beatName){
+/**
+* @param beatName name of the Beat
+* @param arch what architecture
+*/
+def tagAndPush(Map args = [:]) {
+  def beatName = args.beatName
+  def arch = args.get('arch', 'amd64')
   def libbetaVer = env.BEAT_VERSION
   def aliasVersion = ""
   if("${env.SNAPSHOT}" == "true"){
@@ -305,14 +324,22 @@ def tagAndPush(beatName){
 
   dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
 
+  // supported tags
+  def tags = [tagName, "${env.GIT_BASE_COMMIT}"]
+  if (!isPR() && aliasVersion != "") {
+    tags << aliasVersion
+  }
   // supported image flavours
   def variants = ["", "-oss", "-ubi8"]
   variants.each { variant ->
-    doTagAndPush(beatName, variant, libbetaVer, tagName)
-    doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}")
-
-    if (!isPR() && aliasVersion != "") {
-      doTagAndPush(beatName, variant, libbetaVer, aliasVersion)
+    tags.each { tag ->
+      // TODO:
+      // For backward compatibility let's ensure we tag only for amd64, then E2E can benefit from until
+      // they support the versioning with the architecture
+      if ("${arch}" == "amd64") {
+        doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}")
+      }
+      doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}")
     }
   }
 }
@@ -323,18 +350,19 @@ def tagAndPush(beatName){
 * @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
 * @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
 */
-def doTagAndPush(beatName, variant, sourceTag, targetTag) {
+def doTagAndPush(Map args = [:]) {
+  def beatName = args.beatName
+  def variant = args.variant
+  def sourceTag = args.sourceTag
+  def targetTag = args.targetTag
   def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
   def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
-
   def iterations = 0
   retryWithSleep(retries: 3, seconds: 5, backoff: true) {
     iterations++
-    def status = sh(label: "Change tag and push ${targetName}", script: """
-      docker tag ${sourceName} ${targetName}
-      docker push ${targetName}
-    """, returnStatus: true)
-
+    def status = sh(label: "Change tag and push ${targetName}",
+                    script: ".ci/scripts/docker-tag-push.sh ${sourceName} ${targetName}",
+                    returnStatus: true)
     if ( status > 0 && iterations < 3) {
       error("tag and push failed for ${beatName}, retry")
     } else if ( status > 0 ) {
@@ -408,11 +436,6 @@ def triggerE2ETests(String suite) {
     string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
     string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
   ]
-  if (isPR()) {
-    def version = "pr-${env.CHANGE_ID}"
-    parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}"))
-    parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}"))
-  }
 
   build(job: "${e2eTestsPipeline}",
     parameters: parameters,
@@ -448,14 +471,11 @@ def publishPackages(baseDir){
   uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
 }
 
-def uploadPackages(bucketUri, baseDir){
-  googleStorageUpload(bucket: bucketUri,
-    credentialsId: "${JOB_GCS_CREDENTIALS}",
-    pathPrefix: "${baseDir}/build/distributions/",
-    pattern: "${baseDir}/build/distributions/**/*",
-    sharedPublicly: true,
-    showInline: true
-  )
+def uploadPackages(bucketUri, beatsFolder){
+  googleStorageUploadExt(bucket: bucketUri,
+    credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
+    pattern: "${beatsFolder}/build/distributions/**/*",
+    sharedPublicly: true)
 }
 
 /**
@@ -510,4 +530,4 @@ def fixPermissions() {
       }
     }
   }
-}
+}

.ci/scripts/docker-tag-push.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+MSG="parameter missing."
+SOURCE_IMAGE=${1:?$MSG}
+TARGET_IMAGE=${2:?$MSG}
+
+if docker image inspect "${SOURCE_IMAGE}" &> /dev/null ; then
+    docker tag "${SOURCE_IMAGE}" "${TARGET_IMAGE}"
+    docker push "${TARGET_IMAGE}"
+else
+    echo "docker image ${SOURCE_IMAGE} does not exist"
+fi

.ci/scripts/install-docker-compose.sh

@@ -2,12 +2,42 @@
 
 set -exuo pipefail
 
-MSG="parameter missing."
+MSG="environment variable missing: DOCKER_COMPOSE_VERSION."
 DOCKER_COMPOSE_VERSION=${DOCKER_COMPOSE_VERSION:?$MSG}
 HOME=${HOME:?$MSG}
+
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
+if [ "${ARCH}" == "aarch64" ] ; then
+    echo "docker-compose distribution for ARM is not supported yet. See https://github.com/docker/compose/issues/6831."
+    echo "Let's use the installed docker-compose version"
+    exit 0
+fi
+
+if command -v docker-compose
+then
+    set +e
+    echo "Found docker-compose. Checking version.."
+    FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version|awk '{print $3}'|sed s/\,//)
+    if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]
+    then
+        echo "Versions match. No need to install docker-compose. Exiting."
+        exit 0
+    fi
+    set -e
+fi
+
+echo "UNMET DEP: Installing docker-compose"
+
 DC_CMD="${HOME}/bin/docker-compose"
 
 mkdir -p "${HOME}/bin"
 
-curl -sSLo "${DC_CMD}" "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)"
-chmod +x "${DC_CMD}"
+if curl -sSLo "${DC_CMD}" "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)"; then
+    chmod +x "${DC_CMD}"
+else
+    echo "Something bad with the download, let's delete the corrupted binary"
+    if [ -e "${DC_CMD}" ] ; then
+        rm "${DC_CMD}"
+    fi
+    exit 1
+fi

.ci/scripts/install-go.sh

@@ -11,13 +11,15 @@ GVM_CMD="${HOME}/bin/gvm"
 
 if command -v go
 then
+    set +e
     echo "Found Go. Checking version.."
     FOUND_GO_VERSION=$(go version|awk '{print $3}'|sed s/go//)
-    if [ $FOUND_GO_VERSION == $GO_VERSION ]
+    if [ "$FOUND_GO_VERSION" == "$GO_VERSION" ]
     then
         echo "Versions match. No need to install Go. Exiting."
         exit 0
     fi
+    set -e
 fi
 
 if [ "${ARCH}" == "aarch64" ] ; then
@@ -33,9 +35,9 @@ fi
 echo "UNMET DEP: Installing Go"
 mkdir -p "${HOME}/bin"
 
-curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${OS}-${GVM_ARCH_SUFFIX}"
+curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-${OS}-${GVM_ARCH_SUFFIX}"
 chmod +x "${GVM_CMD}"
 
-gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE}
+${GVM_CMD} "${GO_VERSION}" |cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE}
 
-eval $(gvm ${GO_VERSION})
+eval "$("${GVM_CMD}" "${GO_VERSION}")"

.ci/scripts/install-kind.sh

@@ -7,7 +7,37 @@ KIND_VERSION=${KIND_VERSION:?$MSG}
 HOME=${HOME:?$DEFAULT_HOME}
 KIND_CMD="${HOME}/bin/kind"
 
+if command -v kind
+then
+    set +e
+    echo "Found Kind. Checking version.."
+    FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}')
+    if [ "$FOUND_KIND_VERSION" == "$KIND_VERSION" ]
+    then
+        echo "Versions match. No need to install Kind. Exiting."
+        exit 0
+    fi
+    set -e
+fi
+
+echo "UNMET DEP: Installing Kind"
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
+if [ "${ARCH}" == "aarch64" ] ; then
+    ARCH_SUFFIX=arm64
+else
+    ARCH_SUFFIX=amd64
+fi
+
 mkdir -p "${HOME}/bin"
 
-curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
-chmod +x "${KIND_CMD}"
+if curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${OS}-${ARCH_SUFFIX}" ; then
+    chmod +x "${KIND_CMD}"
+else
+    echo "Something bad with the download, let's delete the corrupted binary"
+    if [ -e "${KIND_CMD}" ] ; then
+        rm "${KIND_CMD}"
+    fi
+    exit 1
+fi

.ci/scripts/install-kubectl.sh

@@ -7,8 +7,38 @@ K8S_VERSION=${K8S_VERSION:?$MSG}
 HOME=${HOME:?$DEFAULT_HOME}
 KUBECTL_CMD="${HOME}/bin/kubectl"
 
+if command -v kubectl
+then
+    set +e
+    echo "Found kubectl. Checking version.."
+    FOUND_KUBECTL_VERSION=$(kubectl version --client --short 2>&1 >/dev/null | awk '{print $3}')
+    if [ "${FOUND_KUBECTL_VERSION}" == "${K8S_VERSION}" ]
+    then
+        echo "Versions match. No need to install kubectl. Exiting."
+        exit 0
+    fi
+    set -e
+fi
+
+echo "UNMET DEP: Installing kubectl"
+
 mkdir -p "${HOME}/bin"
 
-curl -sSLo "${KUBECTL_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
-chmod +x "${KUBECTL_CMD}"
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
+if [ "${ARCH}" == "aarch64" ] ; then
+    ARCH_SUFFIX=arm64
+else
+    ARCH_SUFFIX=amd64
+fi
+
+if curl -sSLo "${KUBECTL_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${OS}/${ARCH_SUFFIX}/kubectl" ; then
+    chmod +x "${KUBECTL_CMD}"
+else
+    echo "Something bad with the download, let's delete the corrupted binary"
+    if [ -e "${KUBECTL_CMD}" ] ; then
+        rm "${KUBECTL_CMD}"
+    fi
+    exit 1
+fi