diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index af49e472ce8..c3aff579772 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -58,6 +58,12 @@ if [[ "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
   export API_KEY_TOKEN=$(vault kv get -field apiKey ${CI_ESS_PATH})
   echo ${API_KEY_TOKEN} > ./apiKey
   export TEST_INTEG_AUTH_ESS_APIKEY_FILE=$(realpath ./apiKey)
+
+  # BK analytics
+  echo "--- Prepare BK test analytics token :vault:"
+  BUILDKITE_ANALYTICS_TOKEN=$(vault kv get -field token kv/ci-shared/platform-ingest/buildkite_analytics_token)
+  export BUILDKITE_ANALYTICS_TOKEN
+
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
old mode 100755
new mode 100644
index 8cee27104f8..54db84af7f8
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -26,8 +26,6 @@ if [ -n "$TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE" ]; then
   fi
 fi
 
-source .buildkite/scripts/unset-secrets.sh
-
 if command -v docker &>/dev/null; then
   DOCKER_REGISTRY="docker.elastic.co"
   docker logout $DOCKER_REGISTRY
diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml
index 94b31b9a63b..48b2bf37df9 100644
--- a/.buildkite/integration.pipeline.yml
+++ b/.buildkite/integration.pipeline.yml
@@ -77,6 +77,12 @@ steps:
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Integration tests"
+    plugins:
+      - test-collector#v1.10.1:
+          files: "build/TEST-*.xml"
+          format: "junit"
+          branches: "main"
+          debug: true    
 
   - label: "Serverless Beats Tests"
     depends_on:
@@ -96,3 +102,5 @@ steps:
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"
+
+
diff --git a/.buildkite/scripts/unset-secrets.sh b/.buildkite/scripts/unset-secrets.sh
deleted file mode 100644
index 88815257661..00000000000
--- a/.buildkite/scripts/unset-secrets.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-# Unset all variables ending with _SECRET or _TOKEN
-for var in $(printenv | sed 's;=.*;;' | sort); do
-  if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
-      unset "$var"
-  fi
-done
\ No newline at end of file
diff --git a/.github/workflows/bump-golang.yml b/.github/workflows/bump-golang.yml
index 687948024b3..62a6f625dbc 100644
--- a/.github/workflows/bump-golang.yml
+++ b/.github/workflows/bump-golang.yml
@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@d0950ebbe80f4f80c3392b288d6a218fae872f69 # v0.76.1
+        uses: updatecli/updatecli-action@6b8881a17fc8038e884ec94ff72a49e8e8a4069f # v0.76.1
 
       - name: Run Updatecli in Apply mode
         run: updatecli apply --config .github/updatecli-bump-golang.yml
diff --git a/.mergify.yml b/.mergify.yml
index 81d9f296212..f7b02213b5b 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -332,3 +332,16 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 8.x branch
+    conditions:
+      - merged
+      - label=backport-v8.x
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.x"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
diff --git a/NOTICE.txt b/NOTICE.txt
index b5890b24788..6aa4f255bce 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -8951,11 +8951,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/winlabs/gowin32
-Version: v0.0.0-20221003142512-0d265587d3c9
+Version: v0.0.0-20240828142606-3c8ae9954cab
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/winlabs/gowin32@v0.0.0-20221003142512-0d265587d3c9/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/winlabs/gowin32@v0.0.0-20240828142606-3c8ae9954cab/LICENSE:
 
 
                                  Apache License
diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go
index 6efd816b239..b245c597138 100644
--- a/dev-tools/mage/crossbuild.go
+++ b/dev-tools/mage/crossbuild.go
@@ -33,6 +33,10 @@ var Platforms = BuildPlatforms.Defaults()
 // are considered to be selected (see isPackageTypeSelected).
 var SelectedPackageTypes []PackageType
 
+// SelectedDockerVariants is the list of docker variants. If empty, all docker variants
+// are considered to be selected (see isDockerVariantSelected).
+var SelectedDockerVariants []DockerVariant
+
 func init() {
 	// Allow overriding via PLATFORMS.
 	if expression := os.Getenv("PLATFORMS"); len(expression) > 0 {
@@ -50,6 +54,18 @@ func init() {
 			SelectedPackageTypes = append(SelectedPackageTypes, p)
 		}
 	}
+
+	// Allow overriding via DOCKER_VARIANTS.
+	if dockerVariants := os.Getenv("DOCKER_VARIANTS"); len(dockerVariants) > 0 {
+		for _, docvariant := range strings.Split(dockerVariants, ",") {
+			var v DockerVariant
+			err := v.UnmarshalText([]byte(docvariant))
+			if err != nil {
+				continue
+			}
+			SelectedDockerVariants = append(SelectedDockerVariants, v)
+		}
+	}
 }
 
 // CrossBuildOption defines an option to the CrossBuild target.
diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go
index 87d9e632df1..9948a5dc97b 100644
--- a/dev-tools/mage/dockerbuilder.go
+++ b/dev-tools/mage/dockerbuilder.go
@@ -28,17 +28,12 @@ type dockerBuilder struct {
 }
 
 func newDockerBuilder(spec PackageSpec) (*dockerBuilder, error) {
-	imageName, err := spec.ImageName()
-	if err != nil {
-		return nil, err
-	}
-
 	buildDir := filepath.Join(spec.packageDir, "docker-build")
 	beatDir := filepath.Join(buildDir, "beat")
 
 	return &dockerBuilder{
 		PackageSpec: spec,
-		imageName:   imageName,
+		imageName:   spec.ImageName(),
 		buildDir:    buildDir,
 		beatDir:     beatDir,
 	}, nil
@@ -117,6 +112,7 @@ func (b *dockerBuilder) prepareBuild() error {
 	data := map[string]interface{}{
 		"ExposePorts": b.exposePorts(),
 		"ModulesDirs": b.modulesDirs(),
+		"Variant":     b.DockerVariant.String(),
 	}
 
 	err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {
diff --git a/dev-tools/mage/dockervariants.go b/dev-tools/mage/dockervariants.go
new file mode 100644
index 00000000000..c03f3ba3992
--- /dev/null
+++ b/dev-tools/mage/dockervariants.go
@@ -0,0 +1,78 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package mage
+
+import (
+	"fmt"
+	"strings"
+)
+
+const (
+	undefined = "undefined"
+	basic     = "basic"
+	ubi       = "ubi"
+	wolfi     = "wolfi"
+	complete  = "complete"
+	cloud     = "cloud"
+)
+
+// DockerVariant defines the docker variant to build.
+type DockerVariant int
+
+// List of possible docker variants.
+const (
+	Undefined = iota
+	Basic
+	UBI
+	Wolfi
+	Complete
+	Cloud
+)
+
+// String returns the name of the docker variant type.
+func (typ DockerVariant) String() string {
+	switch typ {
+	case Undefined:
+		return undefined
+	case Basic:
+		return basic
+	case UBI:
+		return ubi
+	case Wolfi:
+		return wolfi
+	case Complete:
+		return complete
+	case Cloud:
+		return cloud
+	default:
+		return invalid
+	}
+}
+
+// MarshalText returns the text representation of DockerVariant.
+func (typ DockerVariant) MarshalText() ([]byte, error) {
+	return []byte(typ.String()), nil
+}
+
+// UnmarshalText returns a DockerVariant based on the given text.
+func (typ *DockerVariant) UnmarshalText(text []byte) error {
+	switch strings.ToLower(string(text)) {
+	case "":
+		*typ = Undefined
+	case basic:
+		*typ = Basic
+	case ubi:
+		*typ = UBI
+	case wolfi:
+		*typ = Wolfi
+	case complete:
+		*typ = Complete
+	case cloud:
+		*typ = Cloud
+	default:
+		return fmt.Errorf("unknown docker variant: %v", string(text))
+	}
+	return nil
+}
diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go
index b13e3d7390f..f728d2b7bd7 100644
--- a/dev-tools/mage/pkg.go
+++ b/dev-tools/mage/pkg.go
@@ -47,6 +47,11 @@ func Package() error {
 					continue
 				}
 
+				if pkgType == Docker && !isDockerVariantSelected(pkg.Spec.DockerVariant) {
+					log.Printf("Skipping %s docker variant type because it is not selected", pkg.Spec.DockerVariant)
+					continue
+				}
+
 				if target.Name == "linux/arm64" && pkgType == Docker && runtime.GOARCH != "arm64" {
 					log.Printf("Skipping Docker package type because build host isn't arm")
 					continue
@@ -121,6 +126,21 @@ func isPackageTypeSelected(pkgType PackageType) bool {
 	return false
 }
 
+// isDockerVariantSelected returns true if SelectedDockerVariants is empty or if
+// docVariant is present on SelectedDockerVariants. It returns false otherwise.
+func isDockerVariantSelected(docVariant DockerVariant) bool {
+	if len(SelectedDockerVariants) == 0 {
+		return true
+	}
+
+	for _, v := range SelectedDockerVariants {
+		if v == docVariant {
+			return true
+		}
+	}
+	return false
+}
+
 type packageBuilder struct {
 	Platform BuildPlatform
 	Spec     PackageSpec
diff --git a/dev-tools/mage/pkgspecs.go b/dev-tools/mage/pkgspecs.go
index c3450c8525f..165984d8735 100644
--- a/dev-tools/mage/pkgspecs.go
+++ b/dev-tools/mage/pkgspecs.go
@@ -154,5 +154,16 @@ func LoadSpecs(files ...string) (map[string][]OSPackageArgs, error) {
 		return nil, fmt.Errorf("failed to unmarshal spec data: %w", err)
 	}
 
+	// verify that the package specification sets the docker variant
+	for specName, specs := range packages.Specs {
+		for _, spec := range specs {
+			for _, pkgType := range spec.Types {
+				if pkgType == Docker && spec.Spec.DockerVariant == Undefined {
+					return nil, fmt.Errorf("%s defined a package spec for docker without a docker_variant set", specName)
+				}
+			}
+		}
+	}
+
 	return packages.Specs, nil
 }
diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go
index fe2d63750ce..ad3892cb198 100644
--- a/dev-tools/mage/pkgtypes.go
+++ b/dev-tools/mage/pkgtypes.go
@@ -91,6 +91,7 @@ type PackageSpec struct {
 	License           string                 `yaml:"license,omitempty"`
 	URL               string                 `yaml:"url,omitempty"`
 	Description       string                 `yaml:"description,omitempty"`
+	DockerVariant     DockerVariant          `yaml:"docker_variant,omitempty"`
 	PreInstallScript  string                 `yaml:"pre_install_script,omitempty"`
 	PostInstallScript string                 `yaml:"post_install_script,omitempty"`
 	PostRmScript      string                 `yaml:"post_rm_script,omitempty"`
@@ -271,11 +272,7 @@ func (typ PackageType) AddFileExtension(file string) string {
 func (typ PackageType) PackagingDir(home string, target BuildPlatform, spec PackageSpec) (string, error) {
 	root := home
 	if typ == Docker {
-		imageName, err := spec.ImageName()
-		if err != nil {
-			return "", err
-		}
-		root = filepath.Join(root, imageName)
+		root = filepath.Join(root, spec.ImageName())
 	}
 
 	targetPath := typ.AddFileExtension(spec.Name + "-" + target.GOOS() + "-" + target.Arch())
@@ -467,17 +464,13 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec {
 	return s
 }
 
-// ImageName computes the image name from the spec. A template for the image
-// name can be configured by adding image_name to extra_vars.
-func (s PackageSpec) ImageName() (string, error) {
-	if name := s.ExtraVars["image_name"]; name != "" {
-		imageName, err := s.Expand(name)
-		if err != nil {
-			return "", fmt.Errorf("failed to expand image_name: %w", err)
-		}
-		return imageName, nil
+// ImageName computes the image name from the spec.
+func (s PackageSpec) ImageName() string {
+	if s.DockerVariant == Basic {
+		// no suffix for basic docker variant
+		return s.Name
 	}
-	return s.Name, nil
+	return fmt.Sprintf("%s-%s", s.Name, s.DockerVariant)
 }
 
 func copyInstallScript(spec PackageSpec, script string, local *string) error {
diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml
index 100a1186b07..786636b0a73 100644
--- a/dev-tools/packaging/packages.yml
+++ b/dev-tools/packaging/packages.yml
@@ -223,31 +223,31 @@ shared:
       buildFrom: '--platform=linux/arm64 cgr.dev/chainguard/wolfi-base'
 
   - &docker_ubuntu_spec
+    docker_variant: 'basic'
     extra_vars:
       from: '--platform=linux/amd64 ubuntu:20.04'
-      image_name: '{{.BeatName}}'
   - &docker_ubuntu_arm_spec
+    docker_variant: 'basic'
     extra_vars:
       from: '--platform=linux/arm64 ubuntu:20.04'
-      image_name: '{{.BeatName}}'
 
   - &docker_ubi_spec
+    docker_variant: 'ubi'
     extra_vars:
       from: '--platform=linux/amd64 docker.elastic.co/ubi9/ubi-minimal'
-      image_name: '{{.BeatName}}-ubi'
   - &docker_ubi_arm_spec
+    docker_variant: 'ubi'
     extra_vars:
       from: '--platform=linux/arm64 docker.elastic.co/ubi9/ubi-minimal'
-      image_name: '{{.BeatName}}-ubi'
 
   - &docker_wolfi_spec
+    docker_variant: 'wolfi'
     extra_vars:
       from: '--platform=linux/amd64 cgr.dev/chainguard/wolfi-base'
-      image_name: '{{.BeatName}}-wolfi'
   - &docker_wolfi_arm_spec
+    docker_variant: 'wolfi'
     extra_vars:
       from: '--platform=linux/arm64 cgr.dev/chainguard/wolfi-base'
-      image_name: '{{.BeatName}}-wolfi'
 
   - &docker_elastic_spec
     extra_vars:
@@ -275,9 +275,10 @@ shared:
           {{ commit }}
         mode: 0644
 
+  # cloud build to beats-ci repository
   - &agent_docker_cloud_spec
+    docker_variant: 'cloud'
     extra_vars:
-      image_name: '{{.BeatName}}-cloud'
       repository: 'docker.elastic.co/beats-ci'
     files:
       'data/cloud_downloads/filebeat.sh':
@@ -290,11 +291,10 @@ shared:
         source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0755
 
-  # not different to the default image, kept for backwards-compatibility
+  # includes nodejs with @elastic/synthetics
   - &agent_docker_complete_spec
     <<: *agent_docker_spec
-    extra_vars:
-      image_name: '{{.BeatName}}-complete'
+    docker_variant: 'complete'
 
   # Deb/RPM spec for community beats.
   - &deb_rpm_spec
diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl
index 9ccb772633d..0e04c3f9ebd 100644
--- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl
+++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl
@@ -43,7 +43,7 @@ RUN true && \
     chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
 {{- end }}
 
-{{- if contains .image_name "-cloud" }}
+{{- if eq .Variant "cloud" }}
     mkdir -p /opt/agentbeat /opt/filebeat /opt/metricbeat && \
     cp -f {{ $beatHome }}/data/cloud_downloads/filebeat.sh /opt/filebeat/filebeat && \
     chmod +x /opt/filebeat/filebeat && \
@@ -170,14 +170,14 @@ RUN mkdir /licenses
 COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses
 COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses
 
-{{- if contains .image_name "-cloud" }}
+{{- if eq .Variant "cloud" }}
 COPY --from=home /opt /opt
 # Generate folder for a stub command that will be overwritten at runtime
 RUN mkdir /app && \
     chown {{ .user }}:{{ .user }} /app
 {{- end }}
 
-{{- if (and (contains .image_name "-complete") (contains .from "ubuntu"))  }}
+{{- if (and (eq .Variant "complete") (contains .from "ubuntu"))  }}
 USER root
 ENV NODE_PATH={{ $beatHome }}/.node
 RUN echo \
@@ -254,7 +254,7 @@ ENV LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE=/
 
 WORKDIR {{ $beatHome }}
 
-{{- if contains .image_name "-cloud" }}
+{{- if eq .Variant "cloud" }}
 ENTRYPOINT ["/usr/bin/tini", "--"]
 CMD ["/app/apm.sh"]
 # Generate a stub command that will be overwritten at runtime
diff --git a/go.mod b/go.mod
index 6168f9c4d6f..c35002dec62 100644
--- a/go.mod
+++ b/go.mod
@@ -54,7 +54,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
 	github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b
-	github.com/winlabs/gowin32 v0.0.0-20221003142512-0d265587d3c9
+	github.com/winlabs/gowin32 v0.0.0-20240828142606-3c8ae9954cab
 	go.elastic.co/apm/module/apmgorilla/v2 v2.6.0
 	go.elastic.co/apm/module/apmgrpc/v2 v2.6.0
 	go.elastic.co/apm/v2 v2.6.0
diff --git a/go.sum b/go.sum
index 4061212bca0..375279d47f8 100644
--- a/go.sum
+++ b/go.sum
@@ -1081,8 +1081,8 @@ github.com/vladimirvivien/gexe v0.2.0 h1:nbdAQ6vbZ+ZNsolCgSVb9Fno60kzSuvtzVh6Ytq
 github.com/vladimirvivien/gexe v0.2.0/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w=
 github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs=
 github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI=
-github.com/winlabs/gowin32 v0.0.0-20221003142512-0d265587d3c9 h1:U8aCPFEMnxAEyj9IonhMVV1gSL4nzelh8uvoXp0hrq0=
-github.com/winlabs/gowin32 v0.0.0-20221003142512-0d265587d3c9/go.mod h1:N51TYkG9JGR5sytj0EoPl31Xg2kuB507lxEmrwSNvfQ=
+github.com/winlabs/gowin32 v0.0.0-20240828142606-3c8ae9954cab h1:jyEvCYk0V6owuSb+Yjpb+qtZjeT04+hTT2CTIPIIa2Q=
+github.com/winlabs/gowin32 v0.0.0-20240828142606-3c8ae9954cab/go.mod h1:N51TYkG9JGR5sytj0EoPl31Xg2kuB507lxEmrwSNvfQ=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
diff --git a/internal/pkg/agent/application/upgrade/upgrade.go b/internal/pkg/agent/application/upgrade/upgrade.go
index 3d1b4fff014..f4cb66e5c47 100644
--- a/internal/pkg/agent/application/upgrade/upgrade.go
+++ b/internal/pkg/agent/application/upgrade/upgrade.go
@@ -353,14 +353,14 @@ func waitForWatcherWithTimeoutCreationFunc(ctx context.Context, log *logger.Logg
 		return fmt.Errorf("error starting update marker watcher: %w", err)
 	}
 
-	log.Info("waiting up to %s for upgrade watcher to set %s state in upgrade marker", waitTime, details.StateWatching)
+	log.Infof("waiting up to %s for upgrade watcher to set %s state in upgrade marker", waitTime, details.StateWatching)
 
 	for {
 		select {
 		case updMarker := <-markerWatcher.Watch():
 			if updMarker.Details != nil && updMarker.Details.State == details.StateWatching {
 				// watcher started and it is watching, all good
-				log.Info("upgrade watcher set %s state in upgrade marker: exiting wait loop", details.StateWatching)
+				log.Infof("upgrade watcher set %s state in upgrade marker: exiting wait loop", details.StateWatching)
 				return nil
 			}
 
diff --git a/pkg/testing/define/define.go b/pkg/testing/define/define.go
index 5804fe84661..8462cf37a0d 100644
--- a/pkg/testing/define/define.go
+++ b/pkg/testing/define/define.go
@@ -17,6 +17,8 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/gofrs/uuid/v5"
+
 	"github.com/elastic/elastic-agent-libs/kibana"
 	"github.com/elastic/go-elasticsearch/v8"
 	"github.com/elastic/go-sysinfo"
@@ -216,28 +218,21 @@ func getOSInfo() (*types.OSInfo, error) {
 // getNamespace is a general namespace that the test can use that will ensure that it
 // is unique and won't collide with other tests (even the same test from a different batch).
 //
-// this function uses a sha256 of the prefix, package and test name, to ensure that the
+// This function uses a sha256 of an UUIDv4 to ensure that the
 // length of the namespace is not over the 100 byte limit from Fleet
 // see: https://www.elastic.co/guide/en/fleet/current/data-streams.html#data-streams-naming-scheme
 func getNamespace(t *testing.T, local bool) (string, error) {
-	prefix := os.Getenv("TEST_DEFINE_PREFIX")
-	if prefix == "" {
-		if local {
-			prefix = "local"
-		}
-		if prefix == "" {
-			return "", errors.New("TEST_DEFINE_PREFIX must be defined by the test runner")
-		}
+	nsUUID, err := uuid.NewV4()
+	if err != nil {
+		return "", fmt.Errorf("cannot generate UUID V4: %w", err)
 	}
-	name := fmt.Sprintf("%s-%s", prefix, t.Name())
 	hasher := sha256.New()
-	hasher.Write([]byte(name))
+	hasher.Write([]byte(nsUUID.String()))
 
 	// Fleet API requires the namespace to be lowercased and not contain
 	// special characters.
 	namespace := strings.ToLower(base64.URLEncoding.EncodeToString(hasher.Sum(nil)))
 	namespace = noSpecialCharsRegexp.ReplaceAllString(namespace, "")
-
 	return namespace, nil
 }
 
diff --git a/pkg/testing/fixture.go b/pkg/testing/fixture.go
index c7338cc63d1..bdd05e32723 100644
--- a/pkg/testing/fixture.go
+++ b/pkg/testing/fixture.go
@@ -206,7 +206,7 @@ func (f *Fixture) Prepare(ctx context.Context, components ...UsableComponent) er
 	if err != nil {
 		return err
 	}
-	workDir := f.t.TempDir()
+	workDir := createTempDir(f.t)
 	finalDir := filepath.Join(workDir, name)
 	err = ExtractArtifact(f.t, src, workDir)
 	if err != nil {
@@ -1196,6 +1196,31 @@ func performConfigure(ctx context.Context, c client.Client, cfg string, timeout
 	return nil
 }
 
+// createTempDir creates a temporary directory that will be
+// removed after the tests passes. If the test fails, the
+// directory is kept for further investigation.
+//
+// If the test is run with -v and fails the temporary directory is logged
+func createTempDir(t *testing.T) string {
+	tempDir, err := os.MkdirTemp("", strings.ReplaceAll(t.Name(), "/", "-"))
+	if err != nil {
+		t.Fatalf("failed to make temp directory: %s", err)
+	}
+
+	cleanup := func() {
+		if !t.Failed() {
+			if err := os.RemoveAll(tempDir); err != nil {
+				t.Errorf("could not remove temp dir '%s': %s", tempDir, err)
+			}
+		} else {
+			t.Logf("Temporary directory %q preserved for investigation/debugging", tempDir)
+		}
+	}
+	t.Cleanup(cleanup)
+
+	return tempDir
+}
+
 type AgentStatusOutput struct {
 	Info struct {
 		ID           string `json:"id"`
diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go
index 1ae0a55b4ec..5430252150f 100644
--- a/pkg/testing/fixture_install.go
+++ b/pkg/testing/fixture_install.go
@@ -402,7 +402,7 @@ func getProcesses(t *gotesting.T, regex string) []runningProcess {
 //   - an error if any.
 func (f *Fixture) installDeb(ctx context.Context, installOpts *InstallOpts, opts []process.CmdOption) ([]byte, error) {
 	f.t.Logf("[test %s] Inside fixture installDeb function", f.t.Name())
-	//Prepare so that the f.srcPackage string is populated
+	// Prepare so that the f.srcPackage string is populated
 	err := f.EnsurePrepared(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("failed to prepare: %w", err)
@@ -416,6 +416,7 @@ func (f *Fixture) installDeb(ctx context.Context, installOpts *InstallOpts, opts
 
 	f.t.Cleanup(func() {
 		f.t.Logf("[test %s] Inside fixture installDeb cleanup function", f.t.Name())
+
 		uninstallCtx, uninstallCancel := context.WithTimeout(context.Background(), 5*time.Minute)
 		defer uninstallCancel()
 		// stop elastic-agent, non fatal if error, might have been stopped before this.
@@ -424,6 +425,12 @@ func (f *Fixture) installDeb(ctx context.Context, installOpts *InstallOpts, opts
 		if err != nil {
 			f.t.Logf("error systemctl stop elastic-agent: %s, output: %s", err, string(out))
 		}
+
+		if keepInstalledFlag() {
+			f.t.Logf("skipping uninstall; test failed and AGENT_KEEP_INSTALLED=true")
+			return
+		}
+
 		// apt-get purge elastic-agent
 		f.t.Logf("running 'sudo apt-get -y -q purge elastic-agent'")
 		out, err = exec.CommandContext(uninstallCtx, "sudo", "apt-get", "-y", "-q", "purge", "elastic-agent").CombinedOutput()
@@ -476,7 +483,7 @@ func (f *Fixture) installDeb(ctx context.Context, installOpts *InstallOpts, opts
 //   - an error if any.
 func (f *Fixture) installRpm(ctx context.Context, installOpts *InstallOpts, opts []process.CmdOption) ([]byte, error) {
 	f.t.Logf("[test %s] Inside fixture installRpm function", f.t.Name())
-	//Prepare so that the f.srcPackage string is populated
+	// Prepare so that the f.srcPackage string is populated
 	err := f.EnsurePrepared(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("failed to prepare: %w", err)
@@ -642,12 +649,12 @@ func (f *Fixture) collectDiagnostics() {
 	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
 	defer cancel()
 
-	dir, err := findProjectRoot(f.caller)
+	diagPath, err := f.DiagDir()
 	if err != nil {
-		f.t.Logf("failed to collect diagnostics; failed to find project root: %s", err)
+		f.t.Logf("failed to collect diagnostics: %v", err)
 		return
 	}
-	diagPath := filepath.Join(dir, "build", "diagnostics")
+
 	err = os.MkdirAll(diagPath, 0755)
 	if err != nil {
 		f.t.Logf("failed to collect diagnostics; failed to create %s: %s", diagPath, err)
@@ -692,6 +699,18 @@ func (f *Fixture) collectDiagnostics() {
 	}
 }
 
+// DiagDir returned {projectRoot}/build/diagnostics path. Files on this path
+// are saved if any test fails. Use it to save files for further investigation.
+func (f *Fixture) DiagDir() (string, error) {
+	dir, err := findProjectRoot(f.caller)
+	if err != nil {
+		return "", fmt.Errorf("failed to find project root: %w", err)
+	}
+
+	diagPath := filepath.Join(dir, "build", "diagnostics")
+	return diagPath, nil
+}
+
 func (f *Fixture) archiveInstallDirectory(installPath string, outputPath string) error {
 	file, err := os.Create(outputPath)
 	if err != nil {
diff --git a/pkg/testing/tools/estools/elasticsearch.go b/pkg/testing/tools/estools/elasticsearch.go
index acce0c38b5a..a766bb5333f 100644
--- a/pkg/testing/tools/estools/elasticsearch.go
+++ b/pkg/testing/tools/estools/elasticsearch.go
@@ -593,6 +593,7 @@ func PerformQueryForRawQuery(ctx context.Context, queryRaw map[string]interface{
 		es.Search.WithContext(ctx),
 		es.Search.WithSize(300),
 	)
+
 	if err != nil {
 		return Documents{}, fmt.Errorf("error performing ES search: %w", err)
 	}
diff --git a/testing/integration/event_logging_test.go b/testing/integration/event_logging_test.go
index f2f0492faeb..aecd1eb9de2 100644
--- a/testing/integration/event_logging_test.go
+++ b/testing/integration/event_logging_test.go
@@ -75,7 +75,7 @@ func TestEventLogFile(t *testing.T) {
 		Local: true,
 		Sudo:  false,
 	})
-
+	t.Skip("Flaky test: https://github.com/elastic/elastic-agent/issues/5397")
 	ctx, cancel := testcontext.WithDeadline(
 		t,
 		context.Background(),
diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go
index 061e03f08e7..81b16715e91 100644
--- a/testing/integration/logs_ingestion_test.go
+++ b/testing/integration/logs_ingestion_test.go
@@ -160,14 +160,15 @@ func testMonitoringLogsAreShipped(
 		return estools.CheckForErrorsInLogs(ctx, info.ESClient, info.Namespace, []string{
 			// acceptable error messages (include reason)
 			"Error dialing dial tcp 127.0.0.1:9200: connect: connection refused", // beat is running default config before its config gets updated
-			"Global configuration artifact is not available",                     // Endpoint: failed to load user artifact due to connectivity issues
+			"Failed to apply initial policy from on disk configuration",
+			"Failed to connect to backoff(elasticsearch(http://127.0.0.1:9200)): Get \"http://127.0.0.1:9200\": dial tcp 127.0.0.1:9200: connect: connection refused", // Deb test
 			"Failed to download artifact",
 			"Failed to initialize artifact",
-			"Failed to apply initial policy from on disk configuration",
-			"elastic-agent-client error: rpc error: code = Canceled desc = context canceled", // can happen on restart
+			"Global configuration artifact is not available",                                 // Endpoint: failed to load user artifact due to connectivity issues
+			"add_cloud_metadata: received error failed fetching EC2 Identity Document",       // okay for the cloud metadata to not work
 			"add_cloud_metadata: received error failed requesting openstack metadata",        // okay for the cloud metadata to not work
 			"add_cloud_metadata: received error failed with http status code 404",            // okay for the cloud metadata to not work
-			"add_cloud_metadata: received error failed fetching EC2 Identity Document",       // okay for the cloud metadata to not work
+			"elastic-agent-client error: rpc error: code = Canceled desc = context canceled", // can happen on restart
 			"failed to invoke rollback watcher: failed to start Upgrade Watcher",             // on debian this happens probably need to fix.
 			"falling back to IMDSv1: operation error ec2imds: getToken",                      // okay for the cloud metadata to not work
 		})
diff --git a/testing/integration/package_version_test.go b/testing/integration/package_version_test.go
index eab8cb4d7d2..08e9ec81b64 100644
--- a/testing/integration/package_version_test.go
+++ b/testing/integration/package_version_test.go
@@ -91,10 +91,11 @@ func TestComponentBuildHashInDiagnostics(t *testing.T) {
 		"failed to install start agent [output: %s]", string(output))
 
 	stateBuff := bytes.Buffer{}
+	var status atesting.AgentStatusOutput
 	allHealthy := func() bool {
 		stateBuff.Reset()
 
-		status, err := f.ExecStatus(ctx)
+		status, err = f.ExecStatus(ctx)
 		if err != nil {
 			stateBuff.WriteString(fmt.Sprintf("failed to get agent status: %v",
 				err))
@@ -102,19 +103,25 @@ func TestComponentBuildHashInDiagnostics(t *testing.T) {
 		}
 
 		for _, c := range status.Components {
+			bs, err := json.MarshalIndent(status, "", "  ")
+			if err != nil {
+				stateBuff.WriteString(fmt.Sprintf("%s not health, could not marshal status outptu: %v",
+					c.Name, err))
+				return false
+			}
+
 			state := client.State(c.State)
 			if state != client.Healthy {
-				bs, err := json.MarshalIndent(status, "", "  ")
-				if err != nil {
-					stateBuff.WriteString(fmt.Sprintf("%s not health, could not marshal status outptu: %v",
-						c.Name, err))
-					return false
-				}
-
 				stateBuff.WriteString(fmt.Sprintf("%s not health, agent status output: %s",
 					c.Name, bs))
 				return false
 			}
+
+			if c.VersionInfo.Meta.Commit == "" {
+				stateBuff.WriteString(fmt.Sprintf("%s health, but no versionInfo. agent status output: %s",
+					c.Name, bs))
+				return false
+			}
 		}
 
 		return true
@@ -123,6 +130,13 @@ func TestComponentBuildHashInDiagnostics(t *testing.T) {
 		allHealthy,
 		5*time.Minute, 10*time.Second,
 		"agent never became healthy. Last status: %v", &stateBuff)
+	t.Cleanup(func() {
+		if !t.Failed() {
+			return
+		}
+
+		t.Logf("test failed: last status output: %v", status)
+	})
 
 	agentbeat := "agentbeat"
 	if runtime.GOOS == "windows" {
@@ -159,6 +173,28 @@ func TestComponentBuildHashInDiagnostics(t *testing.T) {
 
 	diag := t.TempDir()
 	extractZipArchive(t, diagZip, diag)
+	// the test is flaky, so collecting some data to analyze later.
+	t.Cleanup(func() {
+		if !t.Failed() {
+			return
+		}
+
+		t.Logf("the test failed: trying to save the diagnostics used on the test")
+		diagDir, err := f.DiagDir()
+		if err != nil {
+			t.Logf("could not get diagnostics directory to save the diagnostics used on the test")
+			return
+		}
+
+		err = os.Rename(diagZip, filepath.Join(diagDir,
+			fmt.Sprintf("TestComponentBuildHashInDiagnostics-used-diag-%d.zip",
+				time.Now().Unix())))
+		if err != nil {
+			t.Logf("could not move diagnostics used in the test to %s: %v",
+				diagDir, err)
+			return
+		}
+	})
 
 	stateFilePath := filepath.Join(diag, "state.yaml")
 	stateYAML, err := os.Open(stateFilePath)
diff --git a/testing/integration/upgrade_rollback_test.go b/testing/integration/upgrade_rollback_test.go
index b27f74d0b22..1175350bffd 100644
--- a/testing/integration/upgrade_rollback_test.go
+++ b/testing/integration/upgrade_rollback_test.go
@@ -135,7 +135,7 @@ inputs:
 		state, err := client.State(ctx)
 		require.NoError(t, err)
 
-		require.NotNil(t, state.UpgradeDetails)
+		require.NotNil(t, state.UpgradeDetails, "upgrade details in the state cannot be nil")
 		require.Equal(t, details.StateRollback, details.State(state.UpgradeDetails.State))
 	}