Merge branch 'main' into redact-secret-paths-diag

.buildkite/bk.integration.pipeline.yml

@@ -5,7 +5,7 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   ASDF_MAGE_VERSION: 1.14.0
   ASDF_GOLANG_VERSION: 1.22.6
-  ASDF_TERRAFORM_VERSION: 1.9.3
+  ASDF_TERRAFORM_VERSION: 1.9.2
 
 steps:
   - label: "Integration tests: packaging"

.buildkite/pipeline.yml

@@ -158,7 +158,7 @@ steps:
   - label: ":junit: Junit annotate"
     plugins:
       - junit-annotate#v2.5.0:
-          artifacts: "build/*.xml"
+          artifacts: "**TEST-*.xml"
           always-annotate: true
     agents:
       provider: "gcp"

.buildkite/scripts/steps/ess.sh

@@ -13,7 +13,7 @@ function ess_up() {
     return 1
   fi
 
-  export EC_API_KEY=$(retry 5 vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
+  export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
 
   if [[ -z "${EC_API_KEY}" ]]; then
     echo "Error: Failed to get EC API key from vault" >&2
@@ -48,7 +48,7 @@ function ess_down() {
   local WORKSPACE=$(git rev-parse --show-toplevel)
   local TF_DIR="${WORKSPACE}/test_infra/ess/"
   if [ -z "${EC_API_KEY:-}" ]; then
-    export EC_API_KEY=$(retry 5 vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)    
+    export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)    
   fi
 
   pushd "${TF_DIR}"

.buildkite/scripts/sudo-integration-tests.sh

@@ -9,10 +9,13 @@ TESTS_TO_RUN=$2
 
 echo "~~~ Running integration tests as $USER"
 echo "~~~ Integration tests: ${GROUP_NAME}"
-# TODO: Pass the actual version of the agen
 gotestsum --version
+PACKAGE_VERSION="$(cat .package-version)"
+if [[ -n "$PACKAGE_VERSION" ]]; then
+    PACKAGE_VERSION=${PACKAGE_VERSION}"-SNAPSHOT"
+fi
 set +e
-SNAPSHOT=true TEST_DEFINE_PREFIX="sudo_${GROUP_NAME}_ubuntu" gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags integration -test.shuffle on -test.timeout 2h0m0s -test.run "${TESTS_TO_RUN}" github.com/elastic/elastic-agent/testing/integration
+AGENT_VERSION="${PACKAGE_VERSION}" SNAPSHOT=true TEST_DEFINE_PREFIX="sudo_${GROUP_NAME}_ubuntu" gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags integration -test.shuffle on -test.timeout 2h0m0s -test.run "${TESTS_TO_RUN}" github.com/elastic/elastic-agent/testing/integration
 TESTS_EXIT_STATUS=$?
 set -e
 

NOTICE.txt

@@ -1264,11 +1264,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.12.1-0.20241009140206-6634efed32ab
+Version: v0.12.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.12.1-0.20241009140206-6634efed32ab/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.12.1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -2766,11 +2766,11 @@ limitations under the License.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/opentelemetry-collector-components/connector/signaltometricsconnector
-Version: v0.1.0
+Version: v0.2.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/connector/signaltometricsconnector@v0.1.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/connector/signaltometricsconnector@v0.2.1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -3188,11 +3188,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-c
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor
-Version: v0.2.0
+Version: v0.2.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor@v0.2.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor@v0.2.1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -28546,11 +28546,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/lunes@v0.1.0/LI
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/opentelemetry-lib
-Version: v0.11.0
+Version: v0.12.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-lib@v0.11.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-lib@v0.12.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

dev-tools/mage/gotest.go

@@ -5,7 +5,6 @@
 package mage
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -341,42 +340,6 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
 		}
 	}
 
-	// Generate an XML code coverage report.
-	var codecovReport string
-	if params.CoverageProfileFile != "" {
-		fmt.Println(">> go run gocover-cobertura:", params.CoverageProfileFile, "Started")
-
-		// execute gocover-cobertura in order to create cobertura report
-		// install pre-requisites
-		installCobertura := sh.RunCmd("go", "install", "github.com/boumenot/gocover-cobertura@latest")
-		if err = installCobertura(); err != nil {
-			return fmt.Errorf("failed to install gocover-cobertura: %w", err)
-		}
-
-		codecovReport = strings.TrimSuffix(params.CoverageProfileFile,
-			filepath.Ext(params.CoverageProfileFile)) + "-cov.xml"
-
-		coverage, err := os.ReadFile(params.CoverageProfileFile)
-		if err != nil {
-			return fmt.Errorf("failed to read code coverage report: %w", err)
-		}
-
-		coberturaFile, err := os.Create(codecovReport)
-		if err != nil {
-			return err
-		}
-		defer coberturaFile.Close()
-
-		coverToXML := exec.Command("gocover-cobertura")
-		coverToXML.Stdout = coberturaFile
-		coverToXML.Stderr = os.Stderr
-		coverToXML.Stdin = bytes.NewReader(coverage)
-		if err = coverToXML.Run(); err != nil {
-			return fmt.Errorf("failed to write XML code coverage report: %w", err)
-		}
-		fmt.Println(">> go run gocover-cobertura:", params.CoverageProfileFile, "Created")
-	}
-
 	// Return an error indicating that testing failed.
 	if goTestErr != nil {
 		fmt.Println(">> go test:", params.LogName, "Test Failed")

go.mod

@@ -14,17 +14,17 @@ require (
 	github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5
 	github.com/elastic/elastic-agent-autodiscover v0.9.0
 	github.com/elastic/elastic-agent-client/v7 v7.16.0
-	github.com/elastic/elastic-agent-libs v0.12.1-0.20241009140206-6634efed32ab
+	github.com/elastic/elastic-agent-libs v0.12.1
 	github.com/elastic/elastic-agent-system-metrics v0.11.3
 	github.com/elastic/elastic-transport-go/v8 v8.6.0
 	github.com/elastic/go-elasticsearch/v8 v8.15.0
 	github.com/elastic/go-licenser v0.4.2
 	github.com/elastic/go-sysinfo v1.14.2
 	github.com/elastic/go-ucfg v0.8.8
 	github.com/elastic/mock-es v0.0.0-20240712014503-e5b47ece0015
-	github.com/elastic/opentelemetry-collector-components/connector/signaltometricsconnector v0.1.0
+	github.com/elastic/opentelemetry-collector-components/connector/signaltometricsconnector v0.2.1
 	github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor v0.12.0
-	github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor v0.2.0
+	github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor v0.2.1
 	github.com/elastic/opentelemetry-collector-components/processor/lsmintervalprocessor v0.2.0
 	github.com/fatih/color v1.17.0
 	github.com/fsnotify/fsnotify v1.7.0
@@ -196,7 +196,7 @@ require (
 	github.com/elastic/go-windows v1.0.2 // indirect
 	github.com/elastic/gosigar v0.14.3 // indirect
 	github.com/elastic/lunes v0.1.0 // indirect
-	github.com/elastic/opentelemetry-lib v0.11.0 // indirect
+	github.com/elastic/opentelemetry-lib v0.12.0 // indirect
 	github.com/elastic/pkcs8 v1.0.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/envoyproxy/go-control-plane v0.13.0 // indirect

go.sum

@@ -264,8 +264,8 @@ github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0
 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4=
 github.com/elastic/elastic-agent-client/v7 v7.16.0 h1:yKGq2+CxAuW8Kh0EoNl202tqAyQKfBcPRawVKs2Jve0=
 github.com/elastic/elastic-agent-client/v7 v7.16.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
-github.com/elastic/elastic-agent-libs v0.12.1-0.20241009140206-6634efed32ab h1:7XHtHd5w4BV48ToCKGE4NYA0kpI+hv+7S08kdiTjpac=
-github.com/elastic/elastic-agent-libs v0.12.1-0.20241009140206-6634efed32ab/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
+github.com/elastic/elastic-agent-libs v0.12.1 h1:5jkxMx15Bna8cq7/Sz/XUIVUXfNWiJ80iSk4ICQ7KJ0=
+github.com/elastic/elastic-agent-libs v0.12.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
 github.com/elastic/elastic-agent-system-metrics v0.11.3 h1:LDzRwP8kxvsYEtMDgMSKZs1TgPcSEukit+/EAP5Y28A=
 github.com/elastic/elastic-agent-system-metrics v0.11.3/go.mod h1:saqLKe9fuyuAo6IADAnnuy1kaBI7VNlxfwMo8KzSRyQ=
 github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA=
@@ -294,16 +294,16 @@ github.com/elastic/lunes v0.1.0 h1:amRtLPjwkWtzDF/RKzcEPMvSsSseLDLW+bnhfNSLRe4=
 github.com/elastic/lunes v0.1.0/go.mod h1:xGphYIt3XdZRtyWosHQTErsQTd4OP1p9wsbVoHelrd4=
 github.com/elastic/mock-es v0.0.0-20240712014503-e5b47ece0015 h1:z8cC8GASpPo8yKlbnXI36HQ/BM9wYjhBPNbDjAWm0VU=
 github.com/elastic/mock-es v0.0.0-20240712014503-e5b47ece0015/go.mod h1:qH9DX/Dmflz6EAtaks/+2SsdQzecVAKE174Zl66hk7E=
-github.com/elastic/opentelemetry-collector-components/connector/signaltometricsconnector v0.1.0 h1:MdhWZMlkPI62MwS1Jgm7o2H3RSuKsyVKlYlu+17EgBQ=
-github.com/elastic/opentelemetry-collector-components/connector/signaltometricsconnector v0.1.0/go.mod h1:4IROY1OxFvY3e5wFfmvjSO3UcgnnPKp8JDQjBbutzkE=
+github.com/elastic/opentelemetry-collector-components/connector/signaltometricsconnector v0.2.1 h1:OLpob3qEleqQnlgnBRJyClWr63TVThJd5s06CFnzKFI=
+github.com/elastic/opentelemetry-collector-components/connector/signaltometricsconnector v0.2.1/go.mod h1:zIiDkPObWcSOpOzD3iWkaGawMs97oYLFA6i0T5Uv1EI=
 github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor v0.12.0 h1:G5JJsgbpD+8ZXZ17UlDLOgW9E8nwLdmSjm1sppDi/cU=
 github.com/elastic/opentelemetry-collector-components/processor/elasticinframetricsprocessor v0.12.0/go.mod h1:TO8sejwFXzqSyOmL9M8lL6baavxERg9UGJXfYKHdlfA=
-github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor v0.2.0 h1:AQZYupCUbbxFVGAILRH/2E0NHMzM28Vk1Y6JTKyE/7E=
-github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor v0.2.0/go.mod h1:3YI3Xp84W20q7QFVwS7kl54bI0Rjd9Ie58P4GdseqmI=
+github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor v0.2.1 h1:7cCOF19TJPivDc/QjV0VxU1ySwDib6TS2WJ9TP3iEpk=
+github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor v0.2.1/go.mod h1:GSVGd0yJHBXevzOdChtzJFAcrex+iHCNju3fu5QwW14=
 github.com/elastic/opentelemetry-collector-components/processor/lsmintervalprocessor v0.2.0 h1:hS4YRZG4W50od8HMbZM9EQTe0fxWDpCAgo7tAVxuFFE=
 github.com/elastic/opentelemetry-collector-components/processor/lsmintervalprocessor v0.2.0/go.mod h1:/bETRmbfuqPwSI0Y/lVRCGRXhjbnTNKfHZS0NEKffGQ=
-github.com/elastic/opentelemetry-lib v0.11.0 h1:A90YaiYss0wi+NilSOi2mIejqMUVj9oFEi4i8xtbjz0=
-github.com/elastic/opentelemetry-lib v0.11.0/go.mod h1:fdpkzh517xJqSGq3bo/fkdoX/Ag0OoanJoMoIDC3bBk=
+github.com/elastic/opentelemetry-lib v0.12.0 h1:SiCGkT7gLOzkl0wQosQrIWjHWb5eJY18Cm3V3GmdU0o=
+github.com/elastic/opentelemetry-lib v0.12.0/go.mod h1:fdpkzh517xJqSGq3bo/fkdoX/Ag0OoanJoMoIDC3bBk=
 github.com/elastic/pkcs8 v1.0.0 h1:HhitlUKxhN288kcNcYkjW6/ouvuwJWd9ioxpjnD9jVA=
 github.com/elastic/pkcs8 v1.0.0/go.mod h1:ipsZToJfq1MxclVTwpG7U/bgeDtf+0HkUiOxebk95+0=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=

internal/pkg/otel/README.md

@@ -55,7 +55,7 @@ This section provides a summary of components included in the Elastic Distributi
 | Component | Version |
 |---|---|
 | [elasticinframetricsprocessor](https://github.com/elastic/opentelemetry-collector-components/blob/processor/elasticinframetricsprocessor/v0.12.0/processor/elasticinframetricsprocessor/README.md) | v0.12.0 |
-| [elastictraceprocessor](https://github.com/elastic/opentelemetry-collector-components/blob/processor/elastictraceprocessor/v0.2.0/processor/elastictraceprocessor/README.md) | v0.2.0 |
+| [elastictraceprocessor](https://github.com/elastic/opentelemetry-collector-components/blob/processor/elastictraceprocessor/v0.2.1/processor/elastictraceprocessor/README.md) | v0.2.1 |
 | [lsmintervalprocessor](https://github.com/elastic/opentelemetry-collector-components/blob/processor/lsmintervalprocessor/v0.2.0/processor/lsmintervalprocessor/README.md) | v0.2.0 |
 | [memorylimiterprocessor](https://github.com/open-telemetry/opentelemetry-collector/blob/processor/memorylimiterprocessor/v0.111.0/processor/memorylimiterprocessor/README.md) | v0.111.0 |
 | [attributesprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/processor/attributesprocessor/v0.111.0/processor/attributesprocessor/README.md) | v0.111.0 |
@@ -79,5 +79,5 @@ This section provides a summary of components included in the Elastic Distributi
 
 | Component | Version |
 |---|---|
-| [signaltometricsconnector](https://github.com/elastic/opentelemetry-collector-components/blob/connector/signaltometricsconnector/v0.1.0/connector/signaltometricsconnector/README.md) | v0.1.0 |
+| [signaltometricsconnector](https://github.com/elastic/opentelemetry-collector-components/blob/connector/signaltometricsconnector/v0.2.1/connector/signaltometricsconnector/README.md) | v0.2.1 |
 | [spanmetricsconnector](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/connector/spanmetricsconnector/v0.111.0/connector/spanmetricsconnector/README.md) | v0.111.0 |

pkg/testing/windows/windows.go

@@ -40,13 +40,17 @@ func (WindowsRunner) Prepare(ctx context.Context, sshClient ssh.SSHClient, logge
 
 	// install curl
 	logger.Logf("Installing curl")
-	stdOut, errOut, err = sshClient.Exec(ctx, "choco", []string{"install", "-y", "curl"}, nil)
+	curlCtx, curlCancel := context.WithTimeout(ctx, 3*time.Minute)
+	defer curlCancel()
+	stdOut, errOut, err = sshClient.ExecWithRetry(curlCtx, "choco", []string{"install", "-y", "curl"}, 15*time.Second)
 	if err != nil {
 		return fmt.Errorf("failed to install curl: %w (stdout: %s, stderr: %s)", err, stdOut, errOut)
 	}
 	// install make
 	logger.Logf("Installing make")
-	stdOut, errOut, err = sshClient.Exec(ctx, "choco", []string{"install", "-y", "make"}, nil)
+	makeCtx, makeCancel := context.WithTimeout(ctx, 3*time.Minute)
+	defer makeCancel()
+	stdOut, errOut, err = sshClient.ExecWithRetry(makeCtx, "choco", []string{"install", "-y", "make"}, 15*time.Second)
 	if err != nil {
 		return fmt.Errorf("failed to install make: %w (stdout: %s, stderr: %s)", err, stdOut, errOut)
 	}
@@ -55,7 +59,9 @@ func (WindowsRunner) Prepare(ctx context.Context, sshClient ssh.SSHClient, logge
 	logger.Logf("Installing golang %s (%s)", goVersion, arch)
 	downloadURL := fmt.Sprintf("https://go.dev/dl/go%s.windows-%s.msi", goVersion, arch)
 	filename := path.Base(downloadURL)
-	stdOut, errOut, err = sshClient.Exec(ctx, "curl", []string{"-Ls", downloadURL, "--output", filename}, nil)
+	goCtx, goCancel := context.WithTimeout(ctx, 3*time.Minute)
+	defer goCancel()
+	stdOut, errOut, err = sshClient.ExecWithRetry(goCtx, "curl", []string{"-Ls", downloadURL, "--output", filename}, 15*time.Second)
 	if err != nil {
 		return fmt.Errorf("failed to download go from %s with curl: %w (stdout: %s, stderr: %s)", downloadURL, err, stdOut, errOut)
 	}