Fix debian packaging for upgrades

changelog/fragments/1722954476-Fix-loss-of-state.enc-on-upgrade-with-debian-packaging.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Fix loss of state.enc on upgrade with debian packaging
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/5260
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/5101

dev-tools/packaging/packages.yml

@@ -20,6 +20,7 @@ shared:
   # Deb/RPM spec for community beats.
   - &deb_rpm_agent_spec
     <<: *common
+    pre_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/linux/preinstall.sh.tmpl'
     post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/linux/postinstall.sh.tmpl'
     post_rm_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/linux/postrm.sh.tmpl'
     files:

dev-tools/packaging/templates/linux/postinstall.sh.tmpl

@@ -2,42 +2,12 @@
 
 set -e
 
-symlink="/usr/share/elastic-agent/bin/elastic-agent"
-old_agent_dir=""
-
-# check if $symlink exists for the previous install
-# and derive the old agent directory
-if test -L "$symlink"; then
-    resolved_symlink="$(readlink -f -- "$symlink")"
-    # check if it is resolved to non empty string
-    if ! [ -z "$resolved_symlink" ]; then
-        old_agent_dir="$( dirname "$resolved_symlink" )"
-    fi
-fi
-
 commit_hash="{{ commit_short }}"
 version_dir="{{agent_package_version}}{{snapshot_suffix}}"
-
+symlink="/usr/share/elastic-agent/bin/elastic-agent"
 new_agent_dir="/var/lib/elastic-agent/data/elastic-agent-$version_dir-$commit_hash"
 
-# copy the state files if there was a previous agent install
-if ! [ -z "$old_agent_dir" ] && ! [ "$old_agent_dir" -ef "$new_agent_dir" ]; then
-    yml_path="$old_agent_dir/state.yml"
-    enc_path="$old_agent_dir/state.enc"
-    echo "migrate state from $old_agent_dir to $new_agent_dir"
-
-    if test -f "$yml_path"; then
-        echo "found "$yml_path", copy to "$new_agent_dir"."
-        cp "$yml_path" "$new_agent_dir"
-    fi
-
-    if test -f "$enc_path"; then
-        echo "found "$enc_path", copy to "$new_agent_dir"."
-        cp "$enc_path" "$new_agent_dir"
-    fi
-fi
-
-# delete symlink if exists
+# delete $symlink if exists
 if test -L "$symlink"; then
     echo "found symlink $symlink, unlink"
     unlink "$symlink"
@@ -47,5 +17,9 @@ fi
 echo "create symlink "$symlink" to "$new_agent_dir/elastic-agent""
 ln -s "$new_agent_dir/elastic-agent" "$symlink"
 
+# reload systemctl and then restart service
+echo "systemd enable/restart elastic-agent"
 systemctl daemon-reload 2> /dev/null
+systemctl enable elastic-agent 2> /dev/null || true
+systemctl restart elastic-agent 2> /dev/null || true
 exit 0

dev-tools/packaging/templates/linux/preinstall.sh.tmpl

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -e
+
+commit_hash="{{ commit_short }}"
+version_dir="{{agent_package_version}}{{snapshot_suffix}}"
+symlink="/usr/share/elastic-agent/bin/elastic-agent"
+new_agent_dir="/var/lib/elastic-agent/data/elastic-agent-$version_dir-$commit_hash"
+old_agent_dir=""
+
+# upon upgrade we migrate the current symlink to an upgrade symlink as the previous
+# installed version will remove the symlink
+if test -L "$symlink"; then
+    resolved_symlink="$(readlink -- "$symlink")"
+    if ! [ -z "$resolved_symlink" ]; then
+        old_agent_dir="$( dirname "$resolved_symlink" )"
+        echo "previous installation directory $old_agent_dir"
+    else
+        echo "unable to read existing symlink"
+    fi
+
+    # copy the state files if there was a previous agent install
+    if ! [ -z "$old_agent_dir" ] && ! [ "$old_agent_dir" -ef "$new_agent_dir" ]; then
+        yml_path="$old_agent_dir/state.yml"
+        enc_path="$old_agent_dir/state.enc"
+        echo "migrate state from $old_agent_dir to $new_agent_dir"
+
+        if test -f "$yml_path"; then
+            echo "found "$yml_path", copy to "$new_agent_dir"."
+            mkdir -p "$new_agent_dir"
+            cp "$yml_path" "$new_agent_dir"
+        else
+            echo "didn't find $yml_path"
+        fi
+
+        if test -f "$enc_path"; then
+            echo "found "$enc_path", copy to "$new_agent_dir"."
+            mkdir -p "$new_agent_dir"
+            cp "$enc_path" "$new_agent_dir"
+        else
+            echo "didn't find $enc_path"
+        fi
+    fi
+else
+    echo "no previous installation found"
+fi

pkg/testing/fixture_install.go

@@ -408,8 +408,8 @@ func (f *Fixture) installDeb(ctx context.Context, installOpts *InstallOpts, opts
 		return nil, fmt.Errorf("failed to prepare: %w", err)
 	}
 
-	// sudo apt install the deb
-	out, err := exec.CommandContext(ctx, "sudo", "apt", "install", f.srcPackage).CombinedOutput() // #nosec G204 -- Need to pass in name of package
+	// sudo apt-get install the deb
+	out, err := exec.CommandContext(ctx, "sudo", "apt-get", "install", "-y", f.srcPackage).CombinedOutput() // #nosec G204 -- Need to pass in name of package
 	if err != nil {
 		return out, fmt.Errorf("apt install failed: %w output:%s", err, string(out))
 	}

testing/integration/linux_deb_test.go

@@ -0,0 +1,194 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+//go:build integration
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gofrs/uuid/v5"
+
+	"github.com/elastic/elastic-agent-libs/kibana"
+
+	atesting "github.com/elastic/elastic-agent/pkg/testing"
+	"github.com/elastic/elastic-agent/pkg/testing/define"
+	"github.com/elastic/elastic-agent/pkg/testing/tools"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/check"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/fleettools"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/testcontext"
+	"github.com/elastic/elastic-agent/testing/upgradetest"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDebLogIngestFleetManaged(t *testing.T) {
+	info := define.Require(t, define.Requirements{
+		Group: Deb,
+		Stack: &define.Stack{},
+		OS: []define.OS{
+			{
+				Type:   define.Linux,
+				Distro: "ubuntu",
+			},
+		},
+		Local: false,
+		Sudo:  true,
+	})
+
+	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
+	defer cancel()
+
+	agentFixture, err := define.NewFixtureFromLocalBuild(t, define.Version(), atesting.WithPackageFormat("deb"))
+	require.NoError(t, err)
+
+	// 1. Create a policy in Fleet with monitoring enabled.
+	// To ensure there are no conflicts with previous test runs against
+	// the same ESS stack, we add the current time at the end of the policy
+	// name. This policy does not contain any integration.
+	t.Log("Enrolling agent in Fleet with a test policy")
+	createPolicyReq := kibana.AgentPolicy{
+		Name:        fmt.Sprintf("test-policy-enroll-%s", uuid.Must(uuid.NewV4()).String()),
+		Namespace:   info.Namespace,
+		Description: "test policy for agent enrollment",
+		MonitoringEnabled: []kibana.MonitoringEnabledOption{
+			kibana.MonitoringEnabledLogs,
+			kibana.MonitoringEnabledMetrics,
+		},
+		AgentFeatures: []map[string]interface{}{
+			{
+				"name":    "test_enroll",
+				"enabled": true,
+			},
+		},
+	}
+
+	installOpts := atesting.InstallOpts{
+		NonInteractive: true,
+		Force:          true,
+	}
+
+	// 2. Install the Elastic-Agent with the policy that
+	// was just created.
+	policy, err := tools.InstallAgentWithPolicy(
+		ctx,
+		t,
+		installOpts,
+		agentFixture,
+		info.KibanaClient,
+		createPolicyReq)
+	require.NoError(t, err)
+	t.Logf("created policy: %s", policy.ID)
+	check.ConnectedToFleet(ctx, t, agentFixture, 5*time.Minute)
+
+	t.Run("Monitoring logs are shipped", func(t *testing.T) {
+		testMonitoringLogsAreShipped(t, ctx, info, agentFixture, policy)
+	})
+
+	t.Run("Normal logs with flattened data_stream are shipped", func(t *testing.T) {
+		testFlattenedDatastreamFleetPolicy(t, ctx, info, policy)
+	})
+}
+
+func TestDebFleetUpgrade(t *testing.T) {
+	info := define.Require(t, define.Requirements{
+		Group: Deb,
+		Stack: &define.Stack{},
+		OS: []define.OS{
+			{
+				Type:   define.Linux,
+				Distro: "ubuntu",
+			},
+		},
+		Local: false,
+		Sudo:  true,
+	})
+
+	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
+	defer cancel()
+
+	// start from previous minor
+	upgradeFromVersion, err := upgradetest.PreviousMinor()
+	require.NoError(t, err)
+	startFixture, err := atesting.NewFixture(
+		t,
+		upgradeFromVersion.String(),
+		atesting.WithFetcher(atesting.ArtifactFetcher()),
+		atesting.WithPackageFormat("deb"),
+	)
+	require.NoError(t, err)
+
+	// end on the current build with deb
+	endFixture, err := define.NewFixtureFromLocalBuild(t, define.Version(), atesting.WithPackageFormat("deb"))
+	require.NoError(t, err)
+
+	// 1. Create a policy in Fleet with monitoring enabled.
+	// To ensure there are no conflicts with previous test runs against
+	// the same ESS stack, we add the current time at the end of the policy
+	// name. This policy does not contain any integration.
+	t.Log("Enrolling agent in Fleet with a test policy")
+	createPolicyReq := kibana.AgentPolicy{
+		Name:        fmt.Sprintf("test-policy-enroll-%s", uuid.Must(uuid.NewV4()).String()),
+		Namespace:   info.Namespace,
+		Description: "test policy for agent enrollment",
+		MonitoringEnabled: []kibana.MonitoringEnabledOption{
+			kibana.MonitoringEnabledLogs,
+			kibana.MonitoringEnabledMetrics,
+		},
+		AgentFeatures: []map[string]interface{}{
+			{
+				"name":    "test_enroll",
+				"enabled": true,
+			},
+		},
+	}
+
+	installOpts := atesting.InstallOpts{
+		NonInteractive: true,
+		Force:          true,
+	}
+
+	// 2. Install the Elastic-Agent with the policy that
+	// was just created.
+	policy, err := tools.InstallAgentWithPolicy(
+		ctx,
+		t,
+		installOpts,
+		startFixture,
+		info.KibanaClient,
+		createPolicyReq)
+	require.NoError(t, err)
+	t.Logf("created policy: %s", policy.ID)
+	check.ConnectedToFleet(ctx, t, startFixture, 5*time.Minute)
+
+	// 3. Upgrade deb to the build version
+	srcPackage, err := endFixture.SrcPackage(ctx)
+	require.NoError(t, err)
+	cmd := exec.CommandContext(ctx, "sudo", "apt-get", "install", "-y", "-qq", "-o", "Dpkg::Options::=--force-confdef", "-o", "Dpkg::Options::=--force-confold", srcPackage)
+	cmd.Env = append(cmd.Env, "DEBIAN_FRONTEND=noninteractive")
+	out, err := cmd.CombinedOutput() // #nosec G204 -- Need to pass in name of package
+	require.NoError(t, err, string(out))
+
+	// 4. Wait for version in Fleet to match
+	// Fleet will not include the `-SNAPSHOT` in the `GetAgentVersion` result
+	noSnapshotVersion := strings.TrimSuffix(define.Version(), "-SNAPSHOT")
+	require.Eventually(t, func() bool {
+		newVersion, err := fleettools.GetAgentVersion(ctx, info.KibanaClient, policy.ID)
+		if err != nil {
+			t.Logf("error getting agent version: %v", err)
+			return false
+		}
+		if noSnapshotVersion == newVersion {
+			return true
+		}
+		t.Logf("Got Agent version %s != %s", newVersion, noSnapshotVersion)
+		return false
+	}, 5*time.Minute, time.Second)
+}