Merge remote-tracking branch 'upstream/master' into feature/e2e-fleet

* upstream/master: (658 commits)
  Add complete k8s metadata through composable provider (elastic#27691)
  Revert "Fix issue where --insecure didn't propogate to Fleet Server ES connection (elastic#27969)" (elastic#27997)
  Remove deprecated kafka fields (elastic#27938)
  [Filebeat] Add Base64 encoded HMAC & UUID template functions to httpjson input (elastic#27873)
  Improve httpjson template function join (elastic#27996)
  Remove kubernetes.container.image alias (elastic#27898)
  [Elastic Agent] Golden files for program tests (elastic#27862)
  [Elastic Agent] Disable modules.d in metricbeat (elastic#27860)
  libbeat/common/seccomp: provide default policy for linux arm64 (elastic#27955)
  Fix logger statement in aws-s3 input (elastic#27982)
  Fix wrong merge (elastic#27976)
  Fix issue where --insecure didn't propogate to Fleet Server ES connection (elastic#27969)
  Forward-port 7.14.2 changelog to master (elastic#27975)
  [Filebeat] Removing duplicate modules (aliases) Observability (elastic#27919)
  Fix path in vagrant windows script (elastic#27966)
  [Filebeat] Removing duplicate modules (aliases) and Cyberark (elastic#27915)
  No changelog for 8.0.0-alpha2 (elastic#27961)
  Add write access to 'url.value' from 'request.transforms'. (elastic#27937)
  Docker: remove deprecated fields (elastic#27933)
  Filebeat: Make all filesets disabled in default configuration (elastic#27762)
  ...

.backportrc.json

@@ -1,6 +1,6 @@
 {
   "upstream": "elastic/beats",
-  "branches": [ { "name": "7.x", "checked": true }, "7.13", "7.12" ],
+  "branches": [ { "name": "7.x", "checked": true }, "7.15", "7.14", "7.13", "7.12" ],
   "labels": ["backport"],
   "autoAssign": true,
   "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

.ci/apm-beats-update.groovy

@@ -126,7 +126,9 @@ def beatsUpdate() {
         git config --global --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pr/*"
 
         go mod edit -replace github.com/elastic/beats/v7=\${GOPATH}/src/github.com/elastic/beats-local
+        go mod tidy
         echo '{"name": "${GOPATH}/src/github.com/elastic/beats-local", "licenceType": "Elastic"}' >> \${GOPATH}/src/github.com/elastic/beats-local/dev-tools/notice/overrides.json
+
         make update
         git commit -a -m beats-update
 

.ci/bump-go-release-version.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Given the Golang release version this script will bump the version.
+#
+# This script is executed by the automation we are putting in place
+# and it requires the git add/commit commands.
+#
+# Parameters:
+#	$1 -> the Golang release version to be bumped. Mandatory.
+#
+set -euo pipefail
+MSG="parameter missing."
+GO_RELEASE_VERSION=${1:?$MSG}
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+
+if [ "${OS}" == "darwin" ] ; then
+	SED="sed -i .bck"
+else
+	SED="sed -i"
+fi
+
+echo "Update go version ${GO_RELEASE_VERSION}"
+echo "${GO_RELEASE_VERSION}" > .go-version
+git add .go-version
+
+find . -maxdepth 3 -name Dockerfile -print0 |
+    while IFS= read -r -d '' line; do
+        ${SED} -E -e "s#(FROM golang):[0-9]+\.[0-9]+\.[0-9]+#\1:${GO_RELEASE_VERSION}#g" "$line"
+        ${SED} -E -e "s#(ARG GO_VERSION)=[0-9]+\.[0-9]+\.[0-9]+#\1=${GO_RELEASE_VERSION}#g" "$line"
+        git add "${line}"
+    done
+
+${SED} -E -e "s#(:go-version:) [0-9]+\.[0-9]+\.[0-9]+#\1 ${GO_RELEASE_VERSION}#g" libbeat/docs/version.asciidoc
+git add libbeat/docs/version.asciidoc
+
+git diff --staged --quiet || git commit -m "[Automation] Update go release version to ${GO_RELEASE_VERSION}"
+git --no-pager log -1
+
+echo "You can now push and create a Pull Request"

.ci/bump-stack-version.sh

@@ -35,14 +35,15 @@ done
 
 echo "Commit changes"
 if [ "$CREATE_BRANCH" = "true" ]; then
-	git checkout -b "update-stack-version-$(date "+%Y%m%d%H%M%S")"
+	base=$(git rev-parse --abbrev-ref HEAD | sed 's#/#-#g')
+	git checkout -b "update-stack-version-$(date "+%Y%m%d%H%M%S")-${base}"
 else
 	echo "Branch creation disabled."
 fi
 for FILE in ${FILES} ; do
 	git add $FILE
 done
-git diff --staged --quiet || git commit -m "bump stack version ${VERSION}"
+git diff --staged --quiet || git commit -m "[Automation] Update elastic stack version to ${VERSION} for testing"
 git --no-pager log -1
 
 echo "You can now push and create a Pull Request"

.ci/packaging.groovy

@@ -138,7 +138,11 @@ pipeline {
                     'linux/amd64',
                     'linux/386',
                     'linux/arm64',
-                    'linux/armv7',
+                    // armv7 packaging isn't working, and we don't currently
+                    // need it for release. Do not re-enable it without
+                    // confirming it is fixed, you will break the packaging
+                    // pipeline!
+                    //'linux/armv7',
                     // The platforms above are disabled temporarly as crossbuild images are
                     // not available. See: https://github.com/elastic/golang-crossbuild/issues/71
                     //'linux/ppc64le',
@@ -331,6 +335,10 @@ def tagAndPush(Map args = [:]) {
   }
   // supported image flavours
   def variants = ["", "-oss", "-ubi8"]
+  // 
+  if(beatName == 'elastic-agent'){
+      variants.add("-complete")
+  }
   variants.each { variant ->
     tags.each { tag ->
       // TODO:
@@ -425,12 +433,13 @@ def triggerE2ETests(String suite) {
 
   def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
   def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
+  def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"
 
   def parameters = [
     booleanParam(name: 'forceSkipGitChecks', value: true),
     booleanParam(name: 'forceSkipPresubmit', value: true),
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
-    booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
+    string(name: 'BEAT_VERSION', value: beatVersion),
     string(name: 'runTestsSuites', value: suite),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
     string(name: 'GITHUB_CHECK_REPO', value: env.REPO),

.ci/packer_cache.sh

@@ -45,6 +45,8 @@ function dockerPullImages() {
   docker.elastic.co/kibana/kibana:${SNAPSHOT}
   docker.elastic.co/logstash/logstash:${SNAPSHOT}
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm
+  docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armhf
+  docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armel
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-base-arm-debian9
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main

.ci/scripts/generate_build_table.py

@@ -5,8 +5,8 @@
 
 if __name__ == "__main__":
 
-    print("| Beat  | Stage  | Command  | MODULE  | Platforms  | When |")
-    print("|-------|--------|----------|---------|------------|------|")
+    print("| Beat  | Stage  | Category | Command  | MODULE  | Platforms  | When |")
+    print("|-------|--------|----------|----------|---------|------------|------|")
     for root, dirs, files in os.walk("."):
         dirs.sort()
         for file in files:
@@ -18,6 +18,9 @@
                     withModule = False
                     platforms = [doc["platform"]]
                     when = "mandatory"
+                    category = 'default'
+                    if "stage" in doc["stages"][stage]:
+                        category = doc["stages"][stage]["stage"]
                     if "make" in doc["stages"][stage]:
                         command = doc["stages"][stage]["make"].replace("\n", " ")
                     if "mage" in doc["stages"][stage]:
@@ -33,5 +36,5 @@
                     if "when" in doc["stages"][stage]:
                         if "not_changeset_full_match" not in doc["stages"][stage]["when"]:
                             when = "optional"
-                    print("| {} | {} | `{}` | {} | `{}` | {} |".format(
-                        module, stage, command, withModule, platforms, when))
+                    print("| {} | {} | `{}` | `{}` | {} | `{}` | {} |".format(
+                        module, stage, category, command, withModule, platforms, when))

.ci/scripts/install-go.bat

@@ -0,0 +1,57 @@
+set GOPATH=%WORKSPACE%
+set MAGEFILE_CACHE=%WORKSPACE%\.magefile
+
+set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH%
+
+curl --version >nul 2>&1 && (
+    echo found curl
+) || (
+    choco install curl -y --no-progress --skipdownloadcache
+)
+
+mkdir %WORKSPACE%\bin
+
+IF EXIST "%PROGRAMFILES(X86)%" (
+    REM Force the gvm installation.
+    SET GVM_BIN=gvm.exe
+    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe
+    IF ERRORLEVEL 1 (
+        REM gvm installation has failed.
+        del bin\gvm.exe /s /f /q
+        exit /b 1
+    )
+) ELSE (
+    REM Windows 7 workers got a broken gvm installation.
+    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-386.exe
+    IF ERRORLEVEL 1 (
+        REM gvm installation has failed.
+        del bin\gvm.exe /s /f /q
+        exit /b 1
+    )
+)
+
+SET GVM_BIN=gvm.exe
+WHERE /q %GVM_BIN%
+%GVM_BIN% version
+
+REM Install the given go version
+%GVM_BIN% --debug install %GO_VERSION%
+
+REM Configure the given go version
+FOR /f "tokens=*" %%i IN ('"%GVM_BIN%" use %GO_VERSION% --format=batch') DO %%i
+
+go env
+IF ERRORLEVEL 1 (
+    REM go is not configured correctly.
+    rmdir %WORKSPACE%\.gvm /s /q
+    exit /b 1
+)
+
+where mage
+mage -version
+IF ERRORLEVEL 1 (
+    go get github.com/magefile/mage
+    IF ERRORLEVEL 1 (
+        exit /b 1
+    )
+)

.ci/scripts/install-tools.bat

@@ -1,6 +1,3 @@
-set GOPATH=%WORKSPACE%
-set MAGEFILE_CACHE=%WORKSPACE%\.magefile
-
 REM Configure GCC for either 32 or 64 bits
 set MINGW_ARCH=64
 IF NOT EXIST "%PROGRAMFILES(X86)%" (
@@ -14,51 +11,6 @@ curl --version >nul 2>&1 && (
     choco install curl -y --no-progress --skipdownloadcache
 )
 
-mkdir %WORKSPACE%\bin
-
-IF EXIST "%PROGRAMFILES(X86)%" (
-    REM Force the gvm installation.
-    SET GVM_BIN=gvm.exe
-    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe
-    IF ERRORLEVEL 1 (
-        REM gvm installation has failed.
-        del bin\gvm.exe /s /f /q
-        exit /b 1
-    )
-) ELSE (
-    REM Windows 7 workers got a broken gvm installation.
-    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-386.exe
-    IF ERRORLEVEL 1 (
-        REM gvm installation has failed.
-        del bin\gvm.exe /s /f /q
-        exit /b 1
-    )
-)
-
-SET GVM_BIN=gvm.exe
-WHERE /q %GVM_BIN%
-%GVM_BIN% version
-
-REM Install the given go version
-%GVM_BIN% --debug install %GO_VERSION%
-
-REM Configure the given go version
-FOR /f "tokens=*" %%i IN ('"%GVM_BIN%" use %GO_VERSION% --format=batch') DO %%i
-
-go env
-IF ERRORLEVEL 1 (
-    REM go is not configured correctly.
-    rmdir %WORKSPACE%\.gvm /s /q
-    exit /b 1
-)
-
-go get github.com/magefile/mage
-where mage
-mage -version
-IF ERRORLEVEL 1 (
-    exit /b 1
-)
-
 REM Set the USERPROFILE to the previous location to fix issues with chocolatey in windows 2019
 SET PREVIOUS_USERPROFILE=%USERPROFILE%
 SET USERPROFILE=%OLD_USERPROFILE%

.ci/scripts/install-tools.sh

@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-.ci/scripts/install-go.sh
 .ci/scripts/install-docker-compose.sh
 .ci/scripts/install-terraform.sh
-make mage

.ci/scripts/kind-setup.sh

@@ -1,5 +1,23 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-kind create cluster --image kindest/node:${K8S_VERSION}
+kind create cluster --image kindest/node:${K8S_VERSION} --config - <<EOF
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: ClusterConfiguration
+    scheduler:
+      extraArgs:
+        bind-address: "0.0.0.0"
+        port: "10251"
+        secure-port: "10259"
+    controllerManager:
+      extraArgs:
+        bind-address: "0.0.0.0"
+        port: "10252"
+        secure-port: "10257"
+EOF
 kubectl cluster-info

.gitignore

@@ -51,5 +51,5 @@ x-pack/elastic-agent/pkg/agent/operation/tests/scripts/serviceable-1.0-darwin-x8
 *.terraform
 *.tfstate*
 
-# Files generated with the bump elastic stack version automation
-testing/environments/*.bck
+# Files generated with the bump version automations
+*.bck

.go-version

@@ -1 +1 @@
-1.15.9
+1.16.6