Expand mage and add build options

.ci/Jenkinsfile

@@ -78,8 +78,8 @@ pipeline {
             steps {
               withGithubNotify(context: "Test-${PLATFORM}") {
                 dir("${BASE_DIR}"){
-                  withGoEnv(){
-                    goTestJUnit(options: '-v ./...', output: 'junit-report.xml')
+                  withMageEnv(){
+                    cmd(label: 'Go unitTest', script: 'mage unitTest')
                   }
                 }
               }

.gitignore

@@ -47,7 +47,5 @@ fleet.yml
 # agent
 build/
 elastic-agent-shipper
+dist/
 
-
-# VSCode
-/.vscode

.goreleaser.yaml

@@ -0,0 +1,66 @@
+# Make sure to check the documentation at https://goreleaser.com
+before:
+  hooks:
+    - go mod tidy
+    - go generate ./...
+builds:
+  - id: darwin
+    binary: '{{ .ProjectName }}-{{ .Env.DEFAULT_VERSION }}-{{ .Os }}-{{ if eq .Arch "amd64" }}x86_64{{ end }}{{ if eq .Arch "arm64" }}aarch64{{ end }}/{{ .ProjectName }}'
+    goos:
+      - darwin
+    goarch:
+      - amd64
+      - arm64
+    flags:
+      - -buildmode=pie
+    ldflags:
+      - '{{ if eq .Env.DEV "false" }}-s -w{{ end }}'
+      - -X main.Version={{ .Env.DEFAULT_VERSION }} -X main.Commit={{.Commit}} -X main.BuildTime={{.Date}}
+    gcflags:
+      - '{{ if eq .Env.DEV "true" }}all=-N -l{{ end }}'
+    no_unique_dist_dir: true
+  - id: linux
+    binary: '{{ .ProjectName }}-{{ .Env.DEFAULT_VERSION }}-{{ .Os }}-{{ if eq .Arch "amd64" }}x86_64{{ end }}{{ if eq .Arch "386" }}x86{{ end }}{{ if eq .Arch "arm64" }}{{ .Arch }}{{ end }}/{{ .ProjectName }}'
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+      - 386
+    no_unique_dist_dir: true
+    ldflags:
+      - '{{ if eq .Env.DEV "false" }}-s -w{{end}}'
+      - -X main.Version={{ .Env.DEFAULT_VERSION }} -X main.Commit={{.Commit}} -X main.BuildTime={{.Date}}
+    gcflags:
+      - '{{ if eq .Env.DEV "true" }}all=-N -l{{ end }}'
+    overrides:
+      - goos: linux
+        goarch: amd64
+        goarm: ''
+        gomips: ''
+        flags:
+          - -buildmode=pie
+      - goos: linux
+        goarch: arm64
+        goarm: ''
+        gomips: ''
+        flags:
+          - -buildmode=pie
+  - id: windows
+    binary: '{{ .ProjectName }}-{{ .Env.DEFAULT_VERSION }}-{{ .Os }}-{{ if eq .Arch "amd64" }}x86_64{{ end }}{{ if eq .Arch "386" }}x86{{ end }}/{{ .ProjectName }}'
+    goos:
+      - windows
+    goarch:
+      - amd64
+      - 386
+    no_unique_dist_dir: true
+    flags:
+      - -buildmode=pie
+    ldflags:
+      - '{{ if eq .Env.DEV "false" }}-s -w{{ end }}'
+      - -X main.Version={{ .Env.DEFAULT_VERSION }} -X main.Commit={{.Commit}} -X main.BuildTime={{.Date}}
+    gcflags:
+      - '{{ if eq .Env.DEV "true" }}all=-N -l{{ end }}'
+changelog:
+  skip: true
+dist: build/binaries

dev-tools/common/common.go

@@ -0,0 +1,90 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package common
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/magefile/mage/mg"
+	"github.com/pkg/errors"
+
+	"github.com/elastic/elastic-agent-libs/dev-tools/mage/gotool"
+)
+
+const ProjectName = "elastic-agent-shipper"
+
+var PlatformFiles = map[string][]string{
+	"windows":       {"windows-x86_64", "windows-x86"},
+	"linux":         {"linux-arm64", "linux-x86_64", "linux-x86"},
+	"darwin":        {"darwin-aarch64", "darwin-x86_64"},
+	"darwin/amd64":  {"darwin-x86_64"},
+	"darwin/arm64":  {"darwin-aarch64"},
+	"linux/386":     {"linux-x86"},
+	"linux/amd64":   {"linux-x86_64"},
+	"linux/arm64":   {"linux-arm64"},
+	"windows/386":   {"windows-x86"},
+	"windows/amd64": {"windows-x86_64"},
+	"all":           {"darwin-aarch64", "darwin-x86_64", "linux-arm64", "linux-x86_64", "linux-x86", "windows-x86_64", "windows-x86"},
+}
+
+// CreateDir creates the parent directory for the given file.
+func CreateDir(file string) string {
+	// Create the output directory.
+	if dir := filepath.Dir(file); dir != "." {
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			panic(errors.Wrapf(err, "failed to create parent dir for %v", file))
+		}
+	}
+	return file
+}
+
+func InstallGoTestTools() {
+	gotool.Install(gotool.Install.Package("gotest.tools/gotestsum")) //nolint:errcheck //not required
+}
+
+func MakeCommand(ctx context.Context, env map[string]string, cmd string, args ...string) *exec.Cmd {
+	c := exec.CommandContext(ctx, cmd, args...)
+	c.Env = os.Environ()
+	for k, v := range env {
+		c.Env = append(c.Env, k+"="+v)
+	}
+	c.Stdout = ioutil.Discard
+	if mg.Verbose() {
+		c.Stdout = os.Stdout
+	}
+	c.Stderr = os.Stderr
+	c.Stdin = os.Stdin
+	fmt.Println("exec:", cmd, strings.Join(args, " ")) //nolint:forbidigo // just for mage
+	return c
+}
+
+func EnvOrDefault(buildEnv string, defaultValue string) string {
+	if val := os.Getenv(buildEnv); val != "" {
+		boolVal, err := strconv.ParseBool(val)
+		if err != nil {
+			return defaultValue
+		}
+		return strconv.FormatBool(boolVal)
+	}
+	return defaultValue
+}
+
+func BoolEnvOrFalse(buildEnv string) bool {
+	if val := os.Getenv(buildEnv); val != "" {
+		boolVal, err := strconv.ParseBool(val)
+		if err != nil {
+			return false
+		}
+		return boolVal
+	}
+	return false
+}

dev-tools/common/test.go

@@ -0,0 +1,139 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package common
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	"io"
+	"os"
+	"os/exec"
+
+	"github.com/magefile/mage/mg"
+	"github.com/magefile/mage/sh"
+	"github.com/pkg/errors"
+)
+
+func GoUnitTest(ctx context.Context, testCoverage bool) error {
+	mg.Deps(InstallGoTestTools)
+
+	fmt.Println(">> go test:", "Unit Testing") //nolint:forbidigo // just for tests
+	var testArgs []string
+
+	gotestsumArgs := []string{"--no-color"}
+	if mg.Verbose() {
+		gotestsumArgs = append(gotestsumArgs, "-f", "standard-verbose")
+	} else {
+		gotestsumArgs = append(gotestsumArgs, "-f", "standard-quiet")
+	}
+	//create report files
+	fileName := filepath.Join("build", "TEST-go-unit")
+	CreateDir(fileName + ".xml")
+	gotestsumArgs = append(gotestsumArgs, "--junitfile", fileName+".xml")
+	CreateDir(fileName + ".out")
+	gotestsumArgs = append(gotestsumArgs, "--jsonfile", fileName+".out"+".json")
+
+	covFile := fileName + ".cov"
+	if testCoverage {
+		CreateDir(covFile)
+		covFile = CreateDir(filepath.Clean(covFile))
+		testArgs = append(testArgs,
+			"-covermode=atomic",
+			"-coverprofile="+covFile,
+		)
+	}
+
+	testArgs = append(testArgs, []string{"./..."}...)
+
+	args := append(gotestsumArgs, append([]string{"--"}, testArgs...)...)
+
+	goTest := MakeCommand(ctx, map[string]string{}, "gotestsum", args...)
+	// Wire up the outputs.
+	var outputs []io.Writer
+	fileOutput, err := os.Create(CreateDir(fileName + ".out"))
+	if err != nil {
+		return errors.Wrap(err, "failed to create go test output file")
+	}
+	defer fileOutput.Close()
+	outputs = append(outputs, fileOutput)
+
+	output := io.MultiWriter(outputs...)
+	goTest.Stdout = io.MultiWriter(output, os.Stdout)
+	goTest.Stderr = io.MultiWriter(output, os.Stderr)
+	err = goTest.Run()
+
+	var goTestErr *exec.ExitError
+	if err != nil {
+		// Command ran.
+		var exitErr *exec.ExitError
+		if !errors.As(err, &exitErr) {
+			return errors.Wrap(err, "failed to execute go")
+		}
+		// Command ran but failed. Process the output.
+		goTestErr = exitErr
+	}
+
+	// Generate a HTML code coverage report.
+	var htmlCoverReport string
+	if testCoverage {
+		htmlCoverReport = strings.TrimSuffix(covFile,
+			filepath.Ext(covFile)) + ".html"
+		coverToHTML := sh.RunCmd("go", "tool", "cover",
+			"-html="+covFile,
+			"-o", htmlCoverReport)
+		if err = coverToHTML(); err != nil {
+			return errors.Wrap(err, "failed to write HTML code coverage report")
+		}
+	}
+
+	// Generate an XML code coverage report.
+	var codecovReport string
+	if testCoverage {
+		fmt.Println(">> go run gocover-cobertura:", covFile, "Started") //nolint:forbidigo // just for tests
+
+		// execute gocover-cobertura in order to create cobertura report
+		// install pre-requisites
+		installCobertura := sh.RunCmd("go", "install", "github.com/boumenot/gocover-cobertura@latest")
+		if err = installCobertura(); err != nil {
+			return errors.Wrap(err, "failed to install gocover-cobertura")
+		}
+
+		codecovReport = strings.TrimSuffix(covFile,
+			filepath.Ext(covFile)) + "-cov.xml"
+
+		coverage, err := ioutil.ReadFile(covFile)
+		if err != nil {
+			return errors.Wrap(err, "failed to read code coverage report")
+		}
+
+		coberturaFile, err := os.Create(codecovReport)
+		if err != nil {
+			return err
+		}
+		defer coberturaFile.Close()
+
+		coverToXML := exec.Command("gocover-cobertura")
+		coverToXML.Stdout = coberturaFile
+		coverToXML.Stderr = os.Stderr
+		coverToXML.Stdin = bytes.NewReader(coverage)
+		if err = coverToXML.Run(); err != nil {
+			return errors.Wrap(err, "failed to write XML code coverage report")
+		}
+		fmt.Println(">> go run gocover-cobertura:", covFile, "Created") //nolint:forbidigo // just for tests
+	}
+	// Return an error indicating that testing failed.
+	if goTestErr != nil {
+		fmt.Println(">> go test:", "Unit Tests : Test Failed") //nolint:forbidigo // just for tests
+		return errors.Wrap(goTestErr, "go test returned a non-zero value")
+	}
+
+	fmt.Println(">> go test:", "Unit Tests : Test Passed") //nolint:forbidigo // just for tests
+	return nil
+}

dev-tools/templates/notice/overrides.json

@@ -22,4 +22,3 @@
 {"name": "github.com/elastic/e2e-testing", "licenceType": "Elastic"}
 {"name": "github.com/elastic/elastic-agent-shipper-client", "licenceType": "Elastic"}
 {"name": "github.com/awslabs/kinesis-aggregation/go/v2", "licenceType": "Apache-2.0", "url": "https://github.com/awslabs/kinesis-aggregation/blob/master/LICENSE.txt"}
-

go.mod

@@ -17,36 +17,41 @@ require (
 	github.com/elastic/go-ucfg v0.8.6
 	github.com/gofrs/uuid v4.2.0+incompatible
 	github.com/magefile/mage v1.13.0
+	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.7.1
 	go.elastic.co/go-licence-detector v0.5.0
 	golang.org/x/net v0.0.0-20220809184613-07c6da5e1ced
+	gotest.tools/gotestsum v1.7.0
 )
 
 require (
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dnephin/pflag v1.0.7 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/elastic/go-licenser v0.4.1 // indirect
 	github.com/elastic/go-structform v0.0.10 // indirect
 	github.com/elastic/go-sysinfo v1.8.1 // indirect
 	github.com/elastic/go-windows v1.0.1 // indirect
 	github.com/fatih/color v1.13.0 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/gobuffalo/here v0.6.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/licenseclassifier v0.0.0-20200402202327-879cb1424de0 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jcchavezs/porto v0.4.0 // indirect
 	github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
+	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/karrick/godirwalk v1.15.8 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/markbates/pkger v0.17.0 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/pierrec/lz4/v4 v4.1.15 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect
@@ -62,6 +67,7 @@ require (
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
 	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.12 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect