Merge branch 'main' into pkoutsovasilis/agent_helm_chart

.buildkite/hooks/pre-command

@@ -58,6 +58,12 @@ if [[ "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
   export API_KEY_TOKEN=$(vault kv get -field apiKey ${CI_ESS_PATH})
   echo ${API_KEY_TOKEN} > ./apiKey
   export TEST_INTEG_AUTH_ESS_APIKEY_FILE=$(realpath ./apiKey)
+
+  # BK analytics
+  echo "--- Prepare BK test analytics token :vault:"
+  BUILDKITE_ANALYTICS_TOKEN=$(vault kv get -field token kv/ci-shared/platform-ingest/buildkite_analytics_token)
+  export BUILDKITE_ANALYTICS_TOKEN
+
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then

.buildkite/hooks/pre-exit

@@ -26,8 +26,6 @@ if [ -n "$TEST_INTEG_AUTH_GCP_SERVICE_TOKEN_FILE" ]; then
   fi
 fi
 
-source .buildkite/scripts/unset-secrets.sh
-
 if command -v docker &>/dev/null; then
   DOCKER_REGISTRY="docker.elastic.co"
   docker logout $DOCKER_REGISTRY

.buildkite/integration.pipeline.yml

@@ -77,6 +77,12 @@ steps:
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Integration tests"
+    plugins:
+      - test-collector#v1.10.1:
+          files: "build/TEST-*.xml"
+          format: "junit"
+          branches: "main"
+          debug: true    
 
   - label: "Serverless Beats Tests"
     depends_on:
@@ -96,3 +102,5 @@ steps:
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"
+
+

.github/workflows/bump-golang.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@d0950ebbe80f4f80c3392b288d6a218fae872f69 # v0.76.1
+        uses: updatecli/updatecli-action@6b8881a17fc8038e884ec94ff72a49e8e8a4069f # v0.76.1
 
       - name: Run Updatecli in Apply mode
         run: updatecli apply --config .github/updatecli-bump-golang.yml

.mergify.yml

@@ -332,3 +332,16 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 8.x branch
+    conditions:
+      - merged
+      - label=backport-v8.x
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.x"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

NOTICE.txt

@@ -8951,11 +8951,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/winlabs/gowin32
-Version: v0.0.0-20221003142512-0d265587d3c9
+Version: v0.0.0-20240828142606-3c8ae9954cab
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/winlabs/gowin32@v0.0.0-20221003142512-0d265587d3c9/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/winlabs/gowin32@v0.0.0-20240828142606-3c8ae9954cab/LICENSE:
 
 
                                  Apache License

dev-tools/mage/crossbuild.go

@@ -33,6 +33,10 @@ var Platforms = BuildPlatforms.Defaults()
 // are considered to be selected (see isPackageTypeSelected).
 var SelectedPackageTypes []PackageType
 
+// SelectedDockerVariants is the list of docker variants. If empty, all docker variants
+// are considered to be selected (see isDockerVariantSelected).
+var SelectedDockerVariants []DockerVariant
+
 func init() {
 	// Allow overriding via PLATFORMS.
 	if expression := os.Getenv("PLATFORMS"); len(expression) > 0 {
@@ -50,6 +54,18 @@ func init() {
 			SelectedPackageTypes = append(SelectedPackageTypes, p)
 		}
 	}
+
+	// Allow overriding via DOCKER_VARIANTS.
+	if dockerVariants := os.Getenv("DOCKER_VARIANTS"); len(dockerVariants) > 0 {
+		for _, docvariant := range strings.Split(dockerVariants, ",") {
+			var v DockerVariant
+			err := v.UnmarshalText([]byte(docvariant))
+			if err != nil {
+				continue
+			}
+			SelectedDockerVariants = append(SelectedDockerVariants, v)
+		}
+	}
 }
 
 // CrossBuildOption defines an option to the CrossBuild target.

dev-tools/mage/dockerbuilder.go

@@ -28,17 +28,12 @@ type dockerBuilder struct {
 }
 
 func newDockerBuilder(spec PackageSpec) (*dockerBuilder, error) {
-	imageName, err := spec.ImageName()
-	if err != nil {
-		return nil, err
-	}
-
 	buildDir := filepath.Join(spec.packageDir, "docker-build")
 	beatDir := filepath.Join(buildDir, "beat")
 
 	return &dockerBuilder{
 		PackageSpec: spec,
-		imageName:   imageName,
+		imageName:   spec.ImageName(),
 		buildDir:    buildDir,
 		beatDir:     beatDir,
 	}, nil
@@ -117,6 +112,7 @@ func (b *dockerBuilder) prepareBuild() error {
 	data := map[string]interface{}{
 		"ExposePorts": b.exposePorts(),
 		"ModulesDirs": b.modulesDirs(),
+		"Variant":     b.DockerVariant.String(),
 	}
 
 	err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {

dev-tools/mage/dockervariants.go

@@ -0,0 +1,78 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package mage
+
+import (
+	"fmt"
+	"strings"
+)
+
+const (
+	undefined = "undefined"
+	basic     = "basic"
+	ubi       = "ubi"
+	wolfi     = "wolfi"
+	complete  = "complete"
+	cloud     = "cloud"
+)
+
+// DockerVariant defines the docker variant to build.
+type DockerVariant int
+
+// List of possible docker variants.
+const (
+	Undefined = iota
+	Basic
+	UBI
+	Wolfi
+	Complete
+	Cloud
+)
+
+// String returns the name of the docker variant type.
+func (typ DockerVariant) String() string {
+	switch typ {
+	case Undefined:
+		return undefined
+	case Basic:
+		return basic
+	case UBI:
+		return ubi
+	case Wolfi:
+		return wolfi
+	case Complete:
+		return complete
+	case Cloud:
+		return cloud
+	default:
+		return invalid
+	}
+}
+
+// MarshalText returns the text representation of DockerVariant.
+func (typ DockerVariant) MarshalText() ([]byte, error) {
+	return []byte(typ.String()), nil
+}
+
+// UnmarshalText returns a DockerVariant based on the given text.
+func (typ *DockerVariant) UnmarshalText(text []byte) error {
+	switch strings.ToLower(string(text)) {
+	case "":
+		*typ = Undefined
+	case basic:
+		*typ = Basic
+	case ubi:
+		*typ = UBI
+	case wolfi:
+		*typ = Wolfi
+	case complete:
+		*typ = Complete
+	case cloud:
+		*typ = Cloud
+	default:
+		return fmt.Errorf("unknown docker variant: %v", string(text))
+	}
+	return nil
+}

dev-tools/mage/pkg.go

@@ -47,6 +47,11 @@ func Package() error {
 					continue
 				}
 
+				if pkgType == Docker && !isDockerVariantSelected(pkg.Spec.DockerVariant) {
+					log.Printf("Skipping %s docker variant type because it is not selected", pkg.Spec.DockerVariant)
+					continue
+				}
+
 				if target.Name == "linux/arm64" && pkgType == Docker && runtime.GOARCH != "arm64" {
 					log.Printf("Skipping Docker package type because build host isn't arm")
 					continue
@@ -121,6 +126,21 @@ func isPackageTypeSelected(pkgType PackageType) bool {
 	return false
 }
 
+// isDockerVariantSelected returns true if SelectedDockerVariants is empty or if
+// docVariant is present on SelectedDockerVariants. It returns false otherwise.
+func isDockerVariantSelected(docVariant DockerVariant) bool {
+	if len(SelectedDockerVariants) == 0 {
+		return true
+	}
+
+	for _, v := range SelectedDockerVariants {
+		if v == docVariant {
+			return true
+		}
+	}
+	return false
+}
+
 type packageBuilder struct {
 	Platform BuildPlatform
 	Spec     PackageSpec

dev-tools/mage/pkgspecs.go

@@ -154,5 +154,16 @@ func LoadSpecs(files ...string) (map[string][]OSPackageArgs, error) {
 		return nil, fmt.Errorf("failed to unmarshal spec data: %w", err)
 	}
 
+	// verify that the package specification sets the docker variant
+	for specName, specs := range packages.Specs {
+		for _, spec := range specs {
+			for _, pkgType := range spec.Types {
+				if pkgType == Docker && spec.Spec.DockerVariant == Undefined {
+					return nil, fmt.Errorf("%s defined a package spec for docker without a docker_variant set", specName)
+				}
+			}
+		}
+	}
+
 	return packages.Specs, nil
 }

dev-tools/mage/pkgtypes.go

@@ -91,6 +91,7 @@ type PackageSpec struct {
 	License           string                 `yaml:"license,omitempty"`
 	URL               string                 `yaml:"url,omitempty"`
 	Description       string                 `yaml:"description,omitempty"`
+	DockerVariant     DockerVariant          `yaml:"docker_variant,omitempty"`
 	PreInstallScript  string                 `yaml:"pre_install_script,omitempty"`
 	PostInstallScript string                 `yaml:"post_install_script,omitempty"`
 	PostRmScript      string                 `yaml:"post_rm_script,omitempty"`
@@ -271,11 +272,7 @@ func (typ PackageType) AddFileExtension(file string) string {
 func (typ PackageType) PackagingDir(home string, target BuildPlatform, spec PackageSpec) (string, error) {
 	root := home
 	if typ == Docker {
-		imageName, err := spec.ImageName()
-		if err != nil {
-			return "", err
-		}
-		root = filepath.Join(root, imageName)
+		root = filepath.Join(root, spec.ImageName())
 	}
 
 	targetPath := typ.AddFileExtension(spec.Name + "-" + target.GOOS() + "-" + target.Arch())
@@ -467,17 +464,13 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec {
 	return s
 }
 
-// ImageName computes the image name from the spec. A template for the image
-// name can be configured by adding image_name to extra_vars.
-func (s PackageSpec) ImageName() (string, error) {
-	if name := s.ExtraVars["image_name"]; name != "" {
-		imageName, err := s.Expand(name)
-		if err != nil {
-			return "", fmt.Errorf("failed to expand image_name: %w", err)
-		}
-		return imageName, nil
+// ImageName computes the image name from the spec.
+func (s PackageSpec) ImageName() string {
+	if s.DockerVariant == Basic {
+		// no suffix for basic docker variant
+		return s.Name
 	}
-	return s.Name, nil
+	return fmt.Sprintf("%s-%s", s.Name, s.DockerVariant)
 }
 
 func copyInstallScript(spec PackageSpec, script string, local *string) error {

dev-tools/packaging/packages.yml

@@ -223,31 +223,31 @@ shared:
       buildFrom: '--platform=linux/arm64 cgr.dev/chainguard/wolfi-base'
 
   - &docker_ubuntu_spec
+    docker_variant: 'basic'
     extra_vars:
       from: '--platform=linux/amd64 ubuntu:20.04'
-      image_name: '{{.BeatName}}'
   - &docker_ubuntu_arm_spec
+    docker_variant: 'basic'
     extra_vars:
       from: '--platform=linux/arm64 ubuntu:20.04'
-      image_name: '{{.BeatName}}'
 
   - &docker_ubi_spec
+    docker_variant: 'ubi'
     extra_vars:
       from: '--platform=linux/amd64 docker.elastic.co/ubi9/ubi-minimal'
-      image_name: '{{.BeatName}}-ubi'
   - &docker_ubi_arm_spec
+    docker_variant: 'ubi'
     extra_vars:
       from: '--platform=linux/arm64 docker.elastic.co/ubi9/ubi-minimal'
-      image_name: '{{.BeatName}}-ubi'
 
   - &docker_wolfi_spec
+    docker_variant: 'wolfi'
     extra_vars:
       from: '--platform=linux/amd64 cgr.dev/chainguard/wolfi-base'
-      image_name: '{{.BeatName}}-wolfi'
   - &docker_wolfi_arm_spec
+    docker_variant: 'wolfi'
     extra_vars:
       from: '--platform=linux/arm64 cgr.dev/chainguard/wolfi-base'
-      image_name: '{{.BeatName}}-wolfi'
 
   - &docker_elastic_spec
     extra_vars:
@@ -275,9 +275,10 @@ shared:
           {{ commit }}
         mode: 0644
 
+  # cloud build to beats-ci repository
   - &agent_docker_cloud_spec
+    docker_variant: 'cloud'
     extra_vars:
-      image_name: '{{.BeatName}}-cloud'
       repository: 'docker.elastic.co/beats-ci'
     files:
       'data/cloud_downloads/filebeat.sh':
@@ -290,11 +291,10 @@ shared:
         source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0755
 
-  # not different to the default image, kept for backwards-compatibility
+  # includes nodejs with @elastic/synthetics
   - &agent_docker_complete_spec
     <<: *agent_docker_spec
-    extra_vars:
-      image_name: '{{.BeatName}}-complete'
+    docker_variant: 'complete'
 
   # Deb/RPM spec for community beats.
   - &deb_rpm_spec