From 17814cc2d17c42dd644f8e7325fc3c66d5dcecb2 Mon Sep 17 00:00:00 2001 From: Panos Koutsovasilis Date: Tue, 28 Jan 2025 20:46:07 +0200 Subject: [PATCH] Fix Fleet Enrollment Handling for Containerized Agent (#6568) * feat: add k8s integration test to check fleet enrollment * fix: container correct fleet enrollment when token changes or the agent is unenrolled * fix: update TestDiagnosticLocalConfig to include enrollment_token_hash * feat: add a simple retry logic while validate the stored agent api token * feat: add unit-test for shouldFleetEnroll * fix: improve unit-test explicitness and check for expected number of calls * fix: kind in changelog fragment * fix: split up ack-ing fleet in a separate function --- .mockery.yaml | 8 +- ...ken-change-or-the-agent-is-unenrolled.yaml | 32 ++ .../coordinator/diagnostics_test.go | 6 +- internal/pkg/agent/cmd/container.go | 188 ++++++++- internal/pkg/agent/cmd/container_test.go | 286 +++++++++++++ internal/pkg/agent/cmd/enroll_cmd.go | 13 +- internal/pkg/agent/configuration/fleet.go | 11 +- internal/pkg/crypto/hash.go | 64 +++ internal/pkg/crypto/hash_test.go | 47 +++ .../kubernetes_agent_standalone_test.go | 385 +++++++++++++++++- .../handlers/diagnostics_provider_mock.go | 4 +- .../actions/handlers/log_level_setter_mock.go | 2 +- .../actions/handlers/uploader_mock.go | 2 +- .../pkg/agent/application/info/agent_mock.go | 18 +- .../pkg/agent/storage/storage_mock.go | 198 +++++++++ .../pkg/fleetapi/acker/acker_mock.go | 2 +- .../pkg/fleetapi/client/sender_mock.go | 153 +++++++ .../pkg/control/v2/client/client_mock.go | 6 +- 18 files changed, 1376 insertions(+), 49 deletions(-) create mode 100644 changelog/fragments/1737552345-Fix-enrollment-for-containerised-agent-when-there-is-an-enrollement-token-change-or-the-agent-is-unenrolled.yaml create mode 100644 internal/pkg/crypto/hash.go create mode 100644 internal/pkg/crypto/hash_test.go create mode 100644 testing/mocks/internal_/pkg/agent/storage/storage_mock.go create mode 100644 testing/mocks/internal_/pkg/fleetapi/client/sender_mock.go diff --git a/.mockery.yaml b/.mockery.yaml index 42937061c85..c2445f42811 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -8,6 +8,12 @@ packages: github.com/elastic/elastic-agent/pkg/control/v2/client: interfaces: Client: + github.com/elastic/elastic-agent/internal/pkg/fleetapi/client: + interfaces: + Sender: + github.com/elastic/elastic-agent/internal/pkg/agent/storage: + interfaces: + Storage: github.com/elastic/elastic-agent/internal/pkg/agent/application/actions/handlers: interfaces: Uploader: @@ -22,4 +28,4 @@ packages: Acker: github.com/elastic/elastic-agent/internal/pkg/agent/application/info: interfaces: - Agent: \ No newline at end of file + Agent: diff --git a/changelog/fragments/1737552345-Fix-enrollment-for-containerised-agent-when-there-is-an-enrollement-token-change-or-the-agent-is-unenrolled.yaml b/changelog/fragments/1737552345-Fix-enrollment-for-containerised-agent-when-there-is-an-enrollement-token-change-or-the-agent-is-unenrolled.yaml new file mode 100644 index 00000000000..c7ad80de6b7 --- /dev/null +++ b/changelog/fragments/1737552345-Fix-enrollment-for-containerised-agent-when-there-is-an-enrollement-token-change-or-the-agent-is-unenrolled.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: bug-fix + +# Change summary; a 80ish characters long description of the change. +summary: Fix enrollment for containerised agent when enrollment token changes or the agent is unenrolled + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: elastic-agent + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: https://github.com/elastic/elastic-agent/pull/6568 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +issue: https://github.com/elastic/elastic-agent/issues/3586 diff --git a/internal/pkg/agent/application/coordinator/diagnostics_test.go b/internal/pkg/agent/application/coordinator/diagnostics_test.go index 307831c1b50..6e7f4309c85 100644 --- a/internal/pkg/agent/application/coordinator/diagnostics_test.go +++ b/internal/pkg/agent/application/coordinator/diagnostics_test.go @@ -57,8 +57,9 @@ func TestDiagnosticLocalConfig(t *testing.T) { // local-config hook correctly returns it. cfg := &configuration.Configuration{ Fleet: &configuration.FleetAgentConfig{ - Enabled: true, - AccessAPIKey: "test-key", + Enabled: true, + AccessAPIKey: "test-key", + EnrollmentTokenHash: "test-hash", Client: remote.Config{ Protocol: "test-protocol", }, @@ -119,6 +120,7 @@ agent: fleet: enabled: true access_api_key: "test-key" + enrollment_token_hash: "test-hash" agent: protocol: "test-protocol" ` diff --git a/internal/pkg/agent/cmd/container.go b/internal/pkg/agent/cmd/container.go index 1f5a62409b2..697709c90b6 100644 --- a/internal/pkg/agent/cmd/container.go +++ b/internal/pkg/agent/cmd/container.go @@ -6,6 +6,8 @@ package cmd import ( "bytes" + "context" + "encoding/base64" "encoding/json" "fmt" "io" @@ -15,12 +17,14 @@ import ( "os/exec" "path/filepath" "regexp" + "slices" "strconv" "strings" "sync" "syscall" "time" + "github.com/cenkalti/backoff/v4" "github.com/spf13/cobra" "gopkg.in/yaml.v2" @@ -31,8 +35,13 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/agent/configuration" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" + "github.com/elastic/elastic-agent/internal/pkg/agent/storage" "github.com/elastic/elastic-agent/internal/pkg/cli" "github.com/elastic/elastic-agent/internal/pkg/config" + "github.com/elastic/elastic-agent/internal/pkg/crypto" + "github.com/elastic/elastic-agent/internal/pkg/fleetapi" + fleetclient "github.com/elastic/elastic-agent/internal/pkg/fleetapi/client" + "github.com/elastic/elastic-agent/internal/pkg/remote" "github.com/elastic/elastic-agent/pkg/component" "github.com/elastic/elastic-agent/pkg/core/logger" "github.com/elastic/elastic-agent/pkg/core/process" @@ -270,20 +279,9 @@ func containerCmd(streams *cli.IOStreams) error { func runContainerCmd(streams *cli.IOStreams, cfg setupConfig) error { var err error - var client *kibana.Client - executable, err := os.Executable() - if err != nil { - return err - } initTimeout := envTimeout(fleetInitTimeoutName) - _, err = os.Stat(paths.AgentConfigFile()) - if !os.IsNotExist(err) && !cfg.Fleet.Force { - // already enrolled, just run the standard run - return run(containerCfgOverrides, false, initTimeout, isContainer) - } - if cfg.FleetServer.Enable { err = ensureServiceToken(streams, &cfg) if err != nil { @@ -291,15 +289,17 @@ func runContainerCmd(streams *cli.IOStreams, cfg setupConfig) error { } } - if cfg.Fleet.Enroll { + shouldEnroll, err := shouldFleetEnroll(cfg) + if err != nil { + return err + } + if shouldEnroll { var policy *kibanaPolicy token := cfg.Fleet.EnrollmentToken if token == "" && !cfg.FleetServer.Enable { - if client == nil { - client, err = kibanaClient(cfg.Kibana, cfg.Kibana.Headers) - if err != nil { - return err - } + client, err := kibanaClient(cfg.Kibana, cfg.Kibana.Headers) + if err != nil { + return err } policy, err = kibanaFetchPolicy(cfg, client, streams) if err != nil { @@ -318,6 +318,11 @@ func runContainerCmd(streams *cli.IOStreams, cfg setupConfig) error { logInfo(streams, "Policy selected for enrollment: ", policyID) } + executable, err := os.Executable() + if err != nil { + return err + } + cmdArgs, err := buildEnrollArgs(cfg, token, policyID) if err != nil { return err @@ -989,3 +994,152 @@ func isContainer(detail component.PlatformDetail) component.PlatformDetail { detail.OS = component.Container return detail } + +var ( + newFleetClient = func(log *logger.Logger, apiKey string, cfg remote.Config) (fleetclient.Sender, error) { + return fleetclient.NewAuthWithConfig(log, apiKey, cfg) + } + newEncryptedDiskStore = storage.NewEncryptedDiskStore + statAgentConfigFile = os.Stat +) + +// agentInfo implements the AgentInfo interface, and it used in shouldFleetEnroll. +type agentInfo struct { + id string +} + +func (a *agentInfo) AgentID() string { + return a.id +} + +// shouldFleetEnroll returns true if the elastic-agent should enroll to fleet. +func shouldFleetEnroll(setupCfg setupConfig) (bool, error) { + if !setupCfg.Fleet.Enroll { + // Enrollment is explicitly disabled in the setup configuration. + return false, nil + } + + if setupCfg.Fleet.Force { + // Enrollment is explicitly enforced by the setup configuration. + return true, nil + } + + agentCfgFilePath := paths.AgentConfigFile() + _, err := statAgentConfigFile(agentCfgFilePath) + if os.IsNotExist(err) { + // The agent configuration file does not exist, so enrollment is required. + return true, nil + } + + ctx := context.Background() + store, err := newEncryptedDiskStore(ctx, agentCfgFilePath) + if err != nil { + return false, fmt.Errorf("failed to instantiate encrypted disk store: %w", err) + } + + reader, err := store.Load() + if err != nil { + return false, fmt.Errorf("failed to load from disk store: %w", err) + } + + cfg, err := config.NewConfigFrom(reader) + if err != nil { + return false, fmt.Errorf("failed to read from disk store: %w", err) + } + + storedConfig, err := configuration.NewFromConfig(cfg) + if err != nil { + return false, fmt.Errorf("failed to read from disk store: %w", err) + } + + storedFleetHosts := storedConfig.Fleet.Client.GetHosts() + if len(storedFleetHosts) == 0 || !slices.Contains(storedFleetHosts, setupCfg.Fleet.URL) { + // The Fleet URL in the setup does not exist in the stored configuration, so enrollment is required. + return true, nil + } + + // Evaluate the stored enrollment token hash against the setup enrollment token if both are present. + // Note that when "upgrading" from an older agent version the enrollment token hash will not exist + // in the stored configuration. + if len(storedConfig.Fleet.EnrollmentTokenHash) > 0 && len(setupCfg.Fleet.EnrollmentToken) > 0 { + enrollmentHashBytes, err := base64.StdEncoding.DecodeString(storedConfig.Fleet.EnrollmentTokenHash) + if err != nil { + return false, fmt.Errorf("failed to decode hash: %w", err) + } + + err = crypto.ComparePBKDF2HashAndPassword(enrollmentHashBytes, []byte(setupCfg.Fleet.EnrollmentToken)) + switch { + case errors.Is(err, crypto.ErrMismatchedHashAndPassword): + // The stored enrollment token hash does not match the new token, so enrollment is required. + return true, nil + case err != nil: + return false, fmt.Errorf("failed to compare hash: %w", err) + } + } + + // Validate the stored API token to check if the agent is still authorized with Fleet. + log, err := logger.New("fleet_client", false) + if err != nil { + return false, fmt.Errorf("failed to create logger: %w", err) + } + fc, err := newFleetClient(log, storedConfig.Fleet.AccessAPIKey, storedConfig.Fleet.Client) + if err != nil { + return false, fmt.Errorf("failed to create fleet client: %w", err) + } + + // Perform an ACK request with **empty events** to verify the validity of the API token. + // If the agent has been manually un-enrolled through the Kibana UI, the ACK request will fail due to an invalid API token. + // In such cases, the agent should automatically re-enroll and "recover" their enrollment status without manual intervention, + // maintaining seamless operation. + err = ackFleet(ctx, fc, storedConfig.Fleet.Info.ID) + switch { + case errors.Is(err, fleetclient.ErrInvalidAPIKey): + // The API key is invalid, so enrollment is required. + return true, nil + case err != nil: + return false, fmt.Errorf("failed to validate api token: %w", err) + } + + // Update the stored enrollment token hash if there is no previous enrollment token hash + // (can happen when "upgrading" from an older version of the agent) and setup enrollment token is present. + if len(storedConfig.Fleet.EnrollmentTokenHash) == 0 && len(setupCfg.Fleet.EnrollmentToken) > 0 { + enrollmentHashBytes, err := crypto.GeneratePBKDF2FromPassword([]byte(setupCfg.Fleet.EnrollmentToken)) + if err != nil { + return false, errors.New("failed to generate enrollment hash") + } + enrollmentTokenHash := base64.StdEncoding.EncodeToString(enrollmentHashBytes) + storedConfig.Fleet.EnrollmentTokenHash = enrollmentTokenHash + + data, err := yaml.Marshal(storedConfig) + if err != nil { + return false, errors.New("could not marshal config") + } + + if err := safelyStoreAgentInfo(store, bytes.NewReader(data)); err != nil { + return false, fmt.Errorf("failed to store agent config: %w", err) + } + } + + return false, nil +} + +// ackFleet performs an ACK request to the fleet server with **empty events**. +func ackFleet(ctx context.Context, client fleetclient.Sender, agentID string) error { + const retryInterval = time.Second + const maxRetries = 3 + ackRequest := &fleetapi.AckRequest{Events: nil} + ackCMD := fleetapi.NewAckCmd(&agentInfo{agentID}, client) + retries := 0 + return backoff.Retry(func() error { + retries++ + _, err := ackCMD.Execute(ctx, ackRequest) + switch { + case err == nil: + return nil + case errors.Is(err, fleetclient.ErrInvalidAPIKey) || retries == maxRetries: + return backoff.Permanent(err) + default: + return err + } + }, &backoff.ConstantBackOff{Interval: retryInterval}) +} diff --git a/internal/pkg/agent/cmd/container_test.go b/internal/pkg/agent/cmd/container_test.go index 1136985a4e6..7a8ebb6a504 100644 --- a/internal/pkg/agent/cmd/container_test.go +++ b/internal/pkg/agent/cmd/container_test.go @@ -5,18 +5,34 @@ package cmd import ( + "context" + "encoding/base64" "encoding/json" + "errors" + "io" "net/http" "net/http/httptest" + "os" "strings" "testing" "time" + "gopkg.in/yaml.v2" + + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/elastic/elastic-agent-libs/kibana" + "github.com/elastic/elastic-agent/internal/pkg/agent/configuration" + "github.com/elastic/elastic-agent/internal/pkg/agent/storage" "github.com/elastic/elastic-agent/internal/pkg/cli" "github.com/elastic/elastic-agent/internal/pkg/config" + "github.com/elastic/elastic-agent/internal/pkg/crypto" + "github.com/elastic/elastic-agent/internal/pkg/fleetapi/client" + "github.com/elastic/elastic-agent/internal/pkg/remote" + "github.com/elastic/elastic-agent/pkg/core/logger" + mockStorage "github.com/elastic/elastic-agent/testing/mocks/internal_/pkg/agent/storage" + mockFleetClient "github.com/elastic/elastic-agent/testing/mocks/internal_/pkg/fleetapi/client" ) func TestEnvWithDefault(t *testing.T) { @@ -241,3 +257,273 @@ func TestKibanaFetchToken(t *testing.T) { require.Equal(t, "apiKey", ak) }) } + +func TestShouldEnroll(t *testing.T) { + enrollmentToken := "test-token" + enrollmentTokenHash, err := crypto.GeneratePBKDF2FromPassword([]byte(enrollmentToken)) + require.NoError(t, err) + enrollmentTokenHashBase64 := base64.StdEncoding.EncodeToString(enrollmentTokenHash) + + enrollmentTokenOther := "test-token-other" + + fleetNetworkErr := errors.New("fleet network error") + for name, tc := range map[string]struct { + cfg setupConfig + statFn func(path string) (os.FileInfo, error) + encryptedDiskStoreFn func(t *testing.T, savedConfig *configuration.Configuration) storage.Storage + fleetClientFn func(t *testing.T) client.Sender + expectedSavedConfig func(t *testing.T, savedConfig *configuration.Configuration) + expectedShouldEnroll bool + expectedErr error + }{ + "should not enroll if fleet enroll is disabled": { + cfg: setupConfig{Fleet: fleetConfig{Enroll: false}}, + expectedShouldEnroll: false, + }, + "should enroll if fleet force is true": { + cfg: setupConfig{Fleet: fleetConfig{Enroll: true, Force: true}}, + expectedShouldEnroll: true, + }, + "should enroll if config file does not exist": { + statFn: func(path string) (os.FileInfo, error) { return nil, os.ErrNotExist }, + cfg: setupConfig{Fleet: fleetConfig{Enroll: true, Force: true}}, + expectedShouldEnroll: true, + }, + "should enroll on fleet url change": { + statFn: func(path string) (os.FileInfo, error) { return nil, nil }, + cfg: setupConfig{Fleet: fleetConfig{Enroll: true, URL: "host1"}}, + encryptedDiskStoreFn: func(t *testing.T, savedConfig *configuration.Configuration) storage.Storage { + m := mockStorage.NewStorage(t) + m.On("Load").Return(io.NopCloser(strings.NewReader(`fleet: + enabled: true + access_api_key: "test-key" + enrollment_token_hash: "test-hash" + hosts: + - host2 + - host3 + agent: + protocol: "https"`)), nil).Once() + return m + }, + expectedShouldEnroll: true, + }, + "should enroll on fleet token change": { + statFn: func(path string) (os.FileInfo, error) { return nil, nil }, + cfg: setupConfig{Fleet: fleetConfig{Enroll: true, URL: "host1", EnrollmentToken: enrollmentTokenOther}}, + encryptedDiskStoreFn: func(t *testing.T, savedConfig *configuration.Configuration) storage.Storage { + m := mockStorage.NewStorage(t) + m.On("Load").Return(io.NopCloser(strings.NewReader(`fleet: + enabled: true + access_api_key: "test-key" + enrollment_token_hash: "`+enrollmentTokenHashBase64+`" + hosts: + - host1 + - host2 + - host3 + agent: + protocol: "https"`)), nil).Once() + return m + }, + expectedShouldEnroll: true, + }, + "should enroll on unauthorized api": { + statFn: func(path string) (os.FileInfo, error) { return nil, nil }, + cfg: setupConfig{Fleet: fleetConfig{Enroll: true, URL: "host1", EnrollmentToken: enrollmentToken}}, + encryptedDiskStoreFn: func(t *testing.T, savedConfig *configuration.Configuration) storage.Storage { + m := mockStorage.NewStorage(t) + m.On("Load").Return(io.NopCloser(strings.NewReader(`fleet: + enabled: true + access_api_key: "test-key" + enrollment_token_hash: "`+enrollmentTokenHashBase64+`" + hosts: + - host1 + - host2 + - host3 + agent: + protocol: "https"`)), nil).Once() + return m + }, + fleetClientFn: func(t *testing.T) client.Sender { + tries := 0 + m := mockFleetClient.NewSender(t) + call := m.On("Send", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + call.Run(func(args mock.Arguments) { + if tries <= 1 { + call.Return(nil, fleetNetworkErr) + } else { + call.Return(nil, client.ErrInvalidAPIKey) + } + tries++ + }).Times(3) + return m + }, + expectedShouldEnroll: true, + }, + "should not enroll on no changes": { + statFn: func(path string) (os.FileInfo, error) { return nil, nil }, + cfg: setupConfig{Fleet: fleetConfig{Enroll: true, URL: "host1", EnrollmentToken: enrollmentToken}}, + encryptedDiskStoreFn: func(t *testing.T, savedConfig *configuration.Configuration) storage.Storage { + m := mockStorage.NewStorage(t) + m.On("Load").Return(io.NopCloser(strings.NewReader(`fleet: + enabled: true + access_api_key: "test-key" + enrollment_token_hash: "`+enrollmentTokenHashBase64+`" + hosts: + - host1 + - host2 + - host3 + agent: + protocol: "https"`)), nil).Once() + return m + }, + fleetClientFn: func(t *testing.T) client.Sender { + tries := 0 + m := mockFleetClient.NewSender(t) + call := m.On("Send", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + call.Run(func(args mock.Arguments) { + if tries <= 1 { + call.Return(nil, fleetNetworkErr) + } else { + call.Return(&http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(`{"action": "acks", "items":[]}`)), + }, nil) + } + tries++ + }).Times(3) + return m + }, + expectedShouldEnroll: false, + }, + "should fail on fleet network errors": { + statFn: func(path string) (os.FileInfo, error) { return nil, nil }, + cfg: setupConfig{Fleet: fleetConfig{Enroll: true, URL: "host1", EnrollmentToken: enrollmentToken}}, + encryptedDiskStoreFn: func(t *testing.T, savedConfig *configuration.Configuration) storage.Storage { + m := mockStorage.NewStorage(t) + m.On("Load").Return(io.NopCloser(strings.NewReader(`fleet: + enabled: true + access_api_key: "test-key" + enrollment_token_hash: "`+enrollmentTokenHashBase64+`" + hosts: + - host1 + - host2 + - host3 + agent: + protocol: "https"`)), nil).Once() + return m + }, + fleetClientFn: func(t *testing.T) client.Sender { + m := mockFleetClient.NewSender(t) + m.On("Send", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil, fleetNetworkErr).Times(3) + return m + }, + expectedErr: fleetNetworkErr, + }, + "should not update the enrollment token hash if it does not exist in setup configuration": { + statFn: func(path string) (os.FileInfo, error) { return nil, nil }, + cfg: setupConfig{Fleet: fleetConfig{Enroll: true, URL: "host1", EnrollmentToken: ""}}, + encryptedDiskStoreFn: func(t *testing.T, savedConfig *configuration.Configuration) storage.Storage { + m := mockStorage.NewStorage(t) + m.On("Load").Return(io.NopCloser(strings.NewReader(`fleet: + enabled: true + access_api_key: "test-key" + hosts: + - host1 + - host2 + - host3 + agent: + protocol: "https"`)), nil).Once() + return m + }, + fleetClientFn: func(t *testing.T) client.Sender { + m := mockFleetClient.NewSender(t) + m.On("Send", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(&http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(`{"action": "acks", "items":[]}`)), + }, nil).Once() + return m + }, + expectedShouldEnroll: false, + }, + "should not enroll on no changes and update the stored enrollment token hash": { + statFn: func(path string) (os.FileInfo, error) { return nil, nil }, + cfg: setupConfig{Fleet: fleetConfig{Enroll: true, URL: "host1", EnrollmentToken: enrollmentToken}}, + encryptedDiskStoreFn: func(t *testing.T, savedConfig *configuration.Configuration) storage.Storage { + m := mockStorage.NewStorage(t) + m.On("Load").Return(io.NopCloser(strings.NewReader(`fleet: + enabled: true + access_api_key: "test-key" + hosts: + - host1 + - host2 + - host3 + agent: + protocol: "https"`)), nil).Once() + m.On("Save", mock.Anything).Run(func(args mock.Arguments) { + reader := args.Get(0).(io.Reader) + data, _ := io.ReadAll(reader) + _ = yaml.Unmarshal(data, savedConfig) + }).Return(nil).Times(0) + return m + }, + fleetClientFn: func(t *testing.T) client.Sender { + m := mockFleetClient.NewSender(t) + m.On("Send", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(&http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(`{"action": "acks", "items":[]}`)), + }, nil).Once() + return m + }, + expectedSavedConfig: func(t *testing.T, savedConfig *configuration.Configuration) { + require.NotNil(t, savedConfig) + require.NotNil(t, savedConfig.Fleet) + enrollmentTokeHash, err := base64.StdEncoding.DecodeString(savedConfig.Fleet.EnrollmentTokenHash) + require.NoError(t, err) + require.NoError(t, crypto.ComparePBKDF2HashAndPassword(enrollmentTokeHash, []byte(enrollmentToken))) + }, + expectedShouldEnroll: false, + }, + } { + t.Run(name, func(t *testing.T) { + savedConfig := &configuration.Configuration{} + if tc.statFn != nil { + oldStatFn := statAgentConfigFile + statAgentConfigFile = tc.statFn + t.Cleanup(func() { + statAgentConfigFile = oldStatFn + }) + } + if tc.encryptedDiskStoreFn != nil { + oldEncryptedDiskStore := newEncryptedDiskStore + newEncryptedDiskStore = func(ctx context.Context, target string, opts ...storage.EncryptedOptionFunc) (storage.Storage, error) { + return tc.encryptedDiskStoreFn(t, savedConfig), nil + } + t.Cleanup(func() { + newEncryptedDiskStore = oldEncryptedDiskStore + }) + } + if tc.fleetClientFn != nil { + oldFleetClient := newFleetClient + newFleetClient = func(log *logger.Logger, apiKey string, cfg remote.Config) (client.Sender, error) { + return tc.fleetClientFn(t), nil + } + t.Cleanup(func() { + newFleetClient = oldFleetClient + }) + } + actualShouldEnroll, err := shouldFleetEnroll(tc.cfg) + if tc.expectedErr != nil { + require.ErrorIs(t, err, tc.expectedErr) + return + } + require.NoError(t, err) + require.Equal(t, tc.expectedShouldEnroll, actualShouldEnroll) + if tc.expectedSavedConfig != nil { + tc.expectedSavedConfig(t, savedConfig) + } + }) + } +} diff --git a/internal/pkg/agent/cmd/enroll_cmd.go b/internal/pkg/agent/cmd/enroll_cmd.go index 28f5c135794..51d87e0e0bf 100644 --- a/internal/pkg/agent/cmd/enroll_cmd.go +++ b/internal/pkg/agent/cmd/enroll_cmd.go @@ -7,6 +7,7 @@ package cmd import ( "bytes" "context" + "encoding/base64" "fmt" "io" "math/rand/v2" @@ -33,6 +34,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/core/authority" "github.com/elastic/elastic-agent/internal/pkg/core/backoff" monitoringConfig "github.com/elastic/elastic-agent/internal/pkg/core/monitoring/config" + "github.com/elastic/elastic-agent/internal/pkg/crypto" "github.com/elastic/elastic-agent/internal/pkg/fleetapi" fleetclient "github.com/elastic/elastic-agent/internal/pkg/fleetapi/client" "github.com/elastic/elastic-agent/internal/pkg/release" @@ -586,7 +588,7 @@ func (c *enrollCmd) enroll(ctx context.Context, persistentConfig map[string]inte errors.TypeNetwork) } - fleetConfig, err := createFleetConfigFromEnroll(resp.Item.AccessAPIKey, c.remoteConfig) + fleetConfig, err := createFleetConfigFromEnroll(resp.Item.AccessAPIKey, c.options.EnrollAPIKey, c.remoteConfig) if err != nil { return err } @@ -1021,12 +1023,17 @@ func createFleetServerBootstrapConfig( return cfg, nil } -func createFleetConfigFromEnroll(accessAPIKey string, cli remote.Config) (*configuration.FleetAgentConfig, error) { +func createFleetConfigFromEnroll(accessAPIKey string, enrollmentToken string, cli remote.Config) (*configuration.FleetAgentConfig, error) { cfg := configuration.DefaultFleetAgentConfig() cfg.Enabled = true cfg.AccessAPIKey = accessAPIKey cfg.Client = cli - + enrollmentHashBytes, err := crypto.GeneratePBKDF2FromPassword([]byte(enrollmentToken)) + if err != nil { + return nil, errors.New(err, "failed to generate enrollment hash", errors.TypeConfig) + } + enrollmentTokenHash := base64.StdEncoding.EncodeToString(enrollmentHashBytes) + cfg.EnrollmentTokenHash = enrollmentTokenHash if err := cfg.Valid(); err != nil { return nil, errors.New(err, "invalid enrollment options", errors.TypeConfig) } diff --git a/internal/pkg/agent/configuration/fleet.go b/internal/pkg/agent/configuration/fleet.go index dc1741b8694..0020018c20c 100644 --- a/internal/pkg/agent/configuration/fleet.go +++ b/internal/pkg/agent/configuration/fleet.go @@ -12,11 +12,12 @@ import ( // FleetAgentConfig is the internal configuration of the agent after the enrollment is done, // this configuration is not exposed in anyway in the elastic-agent.yml and is only internal configuration. type FleetAgentConfig struct { - Enabled bool `config:"enabled" yaml:"enabled"` - AccessAPIKey string `config:"access_api_key" yaml:"access_api_key"` - Client remote.Config `config:",inline" yaml:",inline"` - Info *AgentInfo `config:"agent" yaml:"agent"` - Server *FleetServerConfig `config:"server" yaml:"server,omitempty"` + Enabled bool `config:"enabled" yaml:"enabled"` + AccessAPIKey string `config:"access_api_key" yaml:"access_api_key"` + EnrollmentTokenHash string `config:"enrollment_token_hash" yaml:"enrollment_token_hash"` + Client remote.Config `config:",inline" yaml:",inline"` + Info *AgentInfo `config:"agent" yaml:"agent"` + Server *FleetServerConfig `config:"server" yaml:"server,omitempty"` } // Valid validates the required fields for accessing the API. diff --git a/internal/pkg/crypto/hash.go b/internal/pkg/crypto/hash.go new file mode 100644 index 00000000000..b88f0d47e34 --- /dev/null +++ b/internal/pkg/crypto/hash.go @@ -0,0 +1,64 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package crypto + +import ( + "bytes" + "crypto/hmac" + "errors" + "fmt" +) + +const ( + // https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#pbkdf2 + hashIterations = 210_000 + hashKeyLength = 64 + hashSaltLength = 16 + hashTotalLength = hashSaltLength + hashKeyLength +) + +// ErrMismatchedHashAndPassword is the error returned from ComparePBKDF2HashAndPassword when a password and hash do +// not match. +var ErrMismatchedHashAndPassword = errors.New("hashedPassword is not the hash of the given password") + +// GeneratePBKDF2FromPassword hashes a password using PBKDF2. +func GeneratePBKDF2FromPassword(password []byte) ([]byte, error) { + // Generate a random salt + salt, err := randomBytes(hashSaltLength) + if err != nil { + return nil, fmt.Errorf("failed to generate salt: %w", err) + } + + // Write hash + // SALT|KEY + key := stretchPassword(password, salt, hashIterations, hashKeyLength) + hash := new(bytes.Buffer) + hash.Write(salt) + hash.Write(key) + + out := hash.Bytes() + if len(out) != hashTotalLength { + return nil, errors.New("written bytes do not match header size") + } + return out, nil +} + +// ComparePBKDF2HashAndPassword verifies if the hashed password matches the provided plain password. +func ComparePBKDF2HashAndPassword(hash []byte, password []byte) error { + if len(hash) != hashTotalLength { + return fmt.Errorf("hashedPassword is invalid") + } + + // Read from hash + // SALT|KEY + salt := hash[:hashSaltLength] + keyFromHash := hash[hashSaltLength:hashTotalLength] + keyFromPassword := stretchPassword(password, salt, hashIterations, hashKeyLength) + if !hmac.Equal(keyFromHash, keyFromPassword) { + return ErrMismatchedHashAndPassword + } + + return nil +} diff --git a/internal/pkg/crypto/hash_test.go b/internal/pkg/crypto/hash_test.go new file mode 100644 index 00000000000..3ef86b17ddb --- /dev/null +++ b/internal/pkg/crypto/hash_test.go @@ -0,0 +1,47 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package crypto + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Fuzzy test for GenerateFromPassword and ComparePBKDF2HashAndPassword +func FuzzGenerateAndCompare(f *testing.F) { + // Seed the fuzzer with a few example passwords + f.Add("password123") + f.Add("123456") + f.Add("!@#$%^&*()_+-=") + f.Add("longpasswordwith1234567890and@symbols") + f.Add("short") + f.Add("V2tzU2c1UUJka2Q5blFyUUJqY1c6V294dlEtWXVUV3FQajZBbzdSd0JWUQ==") + + f.Fuzz(func(t *testing.T, password string) { + if len(password) == 0 { + // Skip empty passwords to avoid unnecessary checks + return + } + + t.Log("Testing password:", password) + + // Generate hashed password + hash, err := GeneratePBKDF2FromPassword([]byte(password)) + if err != nil { + t.Errorf("Failed to generate hashed password: %v", err) + return + } + + // Verify the hashed password + err = ComparePBKDF2HashAndPassword(hash, []byte(password)) + require.NoError(t, err, "Password verification failed") + + // Negative test: modify the password slightly and check verification fails + modifiedPassword := password + "wrong" + err = ComparePBKDF2HashAndPassword(hash, []byte(modifiedPassword)) + require.ErrorIs(t, err, ErrMismatchedHashAndPassword, "Password verification succeeded") + }) +} diff --git a/testing/integration/kubernetes_agent_standalone_test.go b/testing/integration/kubernetes_agent_standalone_test.go index 3e35aa95c0f..280e25e4fbf 100644 --- a/testing/integration/kubernetes_agent_standalone_test.go +++ b/testing/integration/kubernetes_agent_standalone_test.go @@ -16,6 +16,7 @@ import ( "errors" "fmt" "io" + "net/http" "os" "path/filepath" "regexp" @@ -25,6 +26,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/elastic/elastic-agent-libs/kibana" "github.com/elastic/go-elasticsearch/v8" appsv1 "k8s.io/api/apps/v1" @@ -369,6 +371,297 @@ func TestKubernetesAgentHelm(t *testing.T) { k8sStepRunInnerTests("name=agent-pernode-helm-agent", schedulableNodeCount, "agent"), }, }, + { + name: "helm managed agent unenrolled with different enrollment token", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepHelmDeploy(agentK8SHelm, "helm-agent", map[string]any{ + "agent": map[string]any{ + "unprivileged": false, + "image": map[string]any{ + "repository": kCtx.agentImageRepo, + "tag": kCtx.agentImageTag, + "pullPolicy": "Never", + }, + "fleet": map[string]any{ + "enabled": true, + "url": kCtx.enrollParams.FleetURL, + "token": kCtx.enrollParams.EnrollmentToken, + "preset": "perNode", + }, + }, + }), + k8sStepCheckAgentStatus("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", nil), + k8sStepRunInnerTests("name=agent-pernode-helm-agent", schedulableNodeCount, "agent"), + func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) { + // unenroll all agents from fleet and keep track of their ids + unEnrolledIDs := map[string]struct{}{} + k8sStepForEachAgentID("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", func(ctx context.Context, id string) error { + unEnrolledIDs[id] = struct{}{} + _, err = info.KibanaClient.UnEnrollAgent(ctx, kibana.UnEnrollAgentRequest{ + ID: id, + Revoke: true, + }) + return err + })(t, ctx, kCtx, namespace) + k8sStepHelmUninstall("helm-agent")(t, ctx, kCtx, namespace) + + // generate a new enrollment token and re-deploy, the helm chart since it is + // under the same release name and same namespace will have the same state + // as the previous deployment + enrollParams, err := fleettools.NewEnrollParams(ctx, info.KibanaClient) + require.NoError(t, err, "failed to create fleet enroll params") + require.NotEqual(t, kCtx.enrollParams.EnrollmentToken, enrollParams.EnrollmentToken, "enrollment token did not change") + k8sStepHelmDeploy(agentK8SHelm, "helm-agent", map[string]any{ + "agent": map[string]any{ + "unprivileged": false, + "image": map[string]any{ + "repository": kCtx.agentImageRepo, + "tag": kCtx.agentImageTag, + "pullPolicy": "Never", + }, + "fleet": map[string]any{ + "enabled": true, + "url": enrollParams.FleetURL, + "token": enrollParams.EnrollmentToken, + "preset": "perNode", + }, + }, + })(t, ctx, kCtx, namespace) + k8sStepCheckAgentStatus("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", nil)(t, ctx, kCtx, namespace) + k8sStepRunInnerTests("name=agent-pernode-helm-agent", schedulableNodeCount, "agent")(t, ctx, kCtx, namespace) + enrolledIDs := map[string]time.Time{} + k8sStepForEachAgentID("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", func(ctx context.Context, id string) error { + resp, err := kibanaGetAgent(ctx, info.KibanaClient, id) + if err != nil { + return err + } + // no ID should match the ones from the unenrolled ones + if _, exists := unEnrolledIDs[id]; exists { + return fmt.Errorf("agent with id %s found in unEnrolledIDs", id) + } + // keep track of the new enrolled ids and their enrollment time as reported by fleet + enrolledIDs[id] = resp.EnrolledAt + return nil + })(t, ctx, kCtx, namespace) + + // uninstall and reinstall but this time check that the elastic-agent is not re-enrolling + k8sStepHelmUninstall("helm-agent")(t, ctx, kCtx, namespace) + k8sStepHelmDeploy(agentK8SHelm, "helm-agent", map[string]any{ + "agent": map[string]any{ + "unprivileged": false, + "image": map[string]any{ + "repository": kCtx.agentImageRepo, + "tag": kCtx.agentImageTag, + "pullPolicy": "Never", + }, + "fleet": map[string]any{ + "enabled": true, + "url": enrollParams.FleetURL, + "token": enrollParams.EnrollmentToken, + "preset": "perNode", + }, + }, + })(t, ctx, kCtx, namespace) + k8sStepCheckAgentStatus("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", nil)(t, ctx, kCtx, namespace) + k8sStepRunInnerTests("name=agent-pernode-helm-agent", schedulableNodeCount, "agent")(t, ctx, kCtx, namespace) + k8sStepForEachAgentID("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", func(ctx context.Context, id string) error { + resp, err := kibanaGetAgent(ctx, info.KibanaClient, id) + if err != nil { + return err + } + // no ID should match the ones from the unenrolled ones + enrolledAt, exists := enrolledIDs[id] + if !exists { + return fmt.Errorf("agent with id %s not found in enrolledIDs", id) + } + + if !resp.EnrolledAt.Equal(enrolledAt) { + return fmt.Errorf("agent enrollment time is updated") + } + return nil + })(t, ctx, kCtx, namespace) + }, + }, + }, + { + name: "helm managed agent unenrolled", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepHelmDeploy(agentK8SHelm, "helm-agent", map[string]any{ + "agent": map[string]any{ + "unprivileged": false, + "image": map[string]any{ + "repository": kCtx.agentImageRepo, + "tag": kCtx.agentImageTag, + "pullPolicy": "Never", + }, + "fleet": map[string]any{ + "enabled": true, + "url": kCtx.enrollParams.FleetURL, + "token": kCtx.enrollParams.EnrollmentToken, + "preset": "perNode", + }, + }, + }), + k8sStepCheckAgentStatus("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", nil), + k8sStepRunInnerTests("name=agent-pernode-helm-agent", schedulableNodeCount, "agent"), + func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) { + // unenroll all agents from fleet and keep track of their ids + unEnrolledIDs := map[string]struct{}{} + k8sStepForEachAgentID("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", func(ctx context.Context, id string) error { + unEnrolledIDs[id] = struct{}{} + _, err = info.KibanaClient.UnEnrollAgent(ctx, kibana.UnEnrollAgentRequest{ + ID: id, + Revoke: true, + }) + return err + })(t, ctx, kCtx, namespace) + + // re-deploy with the same enrollment token, the helm chart since it is + // under the same release name and same namespace will have the same state + // as the previous deployment + k8sStepHelmUninstall("helm-agent")(t, ctx, kCtx, namespace) + k8sStepHelmDeploy(agentK8SHelm, "helm-agent", map[string]any{ + "agent": map[string]any{ + "unprivileged": false, + "image": map[string]any{ + "repository": kCtx.agentImageRepo, + "tag": kCtx.agentImageTag, + "pullPolicy": "Never", + }, + "fleet": map[string]any{ + "enabled": true, + "url": kCtx.enrollParams.FleetURL, + "token": kCtx.enrollParams.EnrollmentToken, + "preset": "perNode", + }, + }, + })(t, ctx, kCtx, namespace) + k8sStepCheckAgentStatus("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", nil)(t, ctx, kCtx, namespace) + k8sStepRunInnerTests("name=agent-pernode-helm-agent", schedulableNodeCount, "agent")(t, ctx, kCtx, namespace) + enrolledIDs := map[string]time.Time{} + k8sStepForEachAgentID("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", func(ctx context.Context, id string) error { + resp, err := kibanaGetAgent(ctx, info.KibanaClient, id) + if err != nil { + return err + } + // no ID should match the ones from the unenrolled ones + if _, exists := unEnrolledIDs[id]; exists { + return fmt.Errorf("agent with id %s found in unEnrolledIDs", id) + } + // keep track of the new enrolled ids and their enrollment time as reported by fleet + enrolledIDs[id] = resp.EnrolledAt + return nil + })(t, ctx, kCtx, namespace) + + // uninstall and reinstall but this time check that the elastic-agent is not re-enrolling + k8sStepHelmUninstall("helm-agent")(t, ctx, kCtx, namespace) + k8sStepHelmDeploy(agentK8SHelm, "helm-agent", map[string]any{ + "agent": map[string]any{ + "unprivileged": false, + "image": map[string]any{ + "repository": kCtx.agentImageRepo, + "tag": kCtx.agentImageTag, + "pullPolicy": "Never", + }, + "fleet": map[string]any{ + "enabled": true, + "url": kCtx.enrollParams.FleetURL, + "token": kCtx.enrollParams.EnrollmentToken, + "preset": "perNode", + }, + }, + })(t, ctx, kCtx, namespace) + k8sStepCheckAgentStatus("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", nil)(t, ctx, kCtx, namespace) + k8sStepRunInnerTests("name=agent-pernode-helm-agent", schedulableNodeCount, "agent")(t, ctx, kCtx, namespace) + k8sStepForEachAgentID("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", func(ctx context.Context, id string) error { + resp, err := kibanaGetAgent(ctx, info.KibanaClient, id) + if err != nil { + return err + } + // no ID should match the ones from the unenrolled ones + enrolledAt, exists := enrolledIDs[id] + if !exists { + return fmt.Errorf("agent with id %s not found in enrolledIDs", id) + } + + if !resp.EnrolledAt.Equal(enrolledAt) { + return fmt.Errorf("agent enrollment time is updated") + } + return nil + })(t, ctx, kCtx, namespace) + }, + }, + }, + { + name: "helm managed agent upgrade older version", + steps: []k8sTestStep{ + k8sStepCreateNamespace(), + k8sStepHelmDeploy(agentK8SHelm, "helm-agent", map[string]any{ + "agent": map[string]any{ + "unprivileged": false, + "image": map[string]any{ + "repository": kCtx.agentImageRepo, + "tag": "8.17.0", + "pullPolicy": "IfNotPresent", + }, + "fleet": map[string]any{ + "enabled": true, + "url": kCtx.enrollParams.FleetURL, + "token": kCtx.enrollParams.EnrollmentToken, + "preset": "perNode", + }, + }, + }), + k8sStepCheckAgentStatus("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", nil), + func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) { + enrolledIDs := map[string]time.Time{} + k8sStepForEachAgentID("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", func(ctx context.Context, id string) error { + resp, err := kibanaGetAgent(ctx, info.KibanaClient, id) + if err != nil { + return err + } + // keep track of the new enrolled ids and their enrollment time as reported by fleet + enrolledIDs[id] = resp.EnrolledAt + return nil + })(t, ctx, kCtx, namespace) + k8sStepHelmUninstall("helm-agent")(t, ctx, kCtx, namespace) + k8sStepHelmDeploy(agentK8SHelm, "helm-agent", map[string]any{ + "agent": map[string]any{ + "unprivileged": false, + "image": map[string]any{ + "repository": kCtx.agentImageRepo, + "tag": kCtx.agentImageTag, + "pullPolicy": "Never", + }, + "fleet": map[string]any{ + "enabled": true, + "url": kCtx.enrollParams.FleetURL, + "token": kCtx.enrollParams.EnrollmentToken, + "preset": "perNode", + }, + }, + })(t, ctx, kCtx, namespace) + k8sStepCheckAgentStatus("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", nil)(t, ctx, kCtx, namespace) + k8sStepRunInnerTests("name=agent-pernode-helm-agent", schedulableNodeCount, "agent")(t, ctx, kCtx, namespace) + k8sStepForEachAgentID("name=agent-pernode-helm-agent", schedulableNodeCount, "agent", func(ctx context.Context, id string) error { + resp, err := kibanaGetAgent(ctx, info.KibanaClient, id) + if err != nil { + return err + } + enrolledAt, exists := enrolledIDs[id] + if !exists { + return fmt.Errorf("agent with id %s not found in enrolledIDs", id) + } + if !resp.EnrolledAt.Equal(enrolledAt) { + return fmt.Errorf("agent enrollment time is updated") + } + return nil + })(t, ctx, kCtx, namespace) + }, + }, + }, { name: "helm managed agent default kubernetes unprivileged", steps: []k8sTestStep{ @@ -553,6 +846,25 @@ func k8sCheckAgentStatus(ctx context.Context, client klient.Client, stdout *byte } } +// k8sGetAgentID returns the agent ID for the given agent pod +func k8sGetAgentID(ctx context.Context, client klient.Client, stdout *bytes.Buffer, stderr *bytes.Buffer, + namespace string, agentPodName string, containerName string) (string, error) { + command := []string{"elastic-agent", "status", "--output=json"} + + status := atesting.AgentStatusOutput{} // clear status output + stdout.Reset() + stderr.Reset() + if err := client.Resources().ExecInPod(ctx, namespace, agentPodName, containerName, command, stdout, stderr); err != nil { + return "", err + } + + if err := json.Unmarshal(stdout.Bytes(), &status); err != nil { + return "", err + } + + return status.Info.ID, nil +} + // getAgentComponentState returns the component state for the given component name and a bool indicating if it exists. func getAgentComponentState(status atesting.AgentStatusOutput, componentName string) (int, bool) { for _, comp := range status.Components { @@ -1204,6 +1516,25 @@ func k8sStepCheckAgentStatus(agentPodLabelSelector string, expectedPodNumber int } } +func k8sStepForEachAgentID(agentPodLabelSelector string, expectedPodNumber int, containerName string, cb func(ctx context.Context, id string) error) k8sTestStep { + return func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) { + perNodePodList := &corev1.PodList{} + err := kCtx.client.Resources(namespace).List(ctx, perNodePodList, func(opt *metav1.ListOptions) { + opt.LabelSelector = agentPodLabelSelector + }) + require.NoError(t, err, "failed to list pods with selector ", perNodePodList) + require.NotEmpty(t, perNodePodList.Items, "no pods found with selector ", perNodePodList) + require.Equal(t, expectedPodNumber, len(perNodePodList.Items), "unexpected number of pods found with selector ", perNodePodList) + var stdout, stderr bytes.Buffer + for _, pod := range perNodePodList.Items { + id, err := k8sGetAgentID(ctx, kCtx.client, &stdout, &stderr, namespace, pod.Name, containerName) + require.NoError(t, err, "failed to unenroll agent %s", pod.Name) + require.NotEmpty(t, id, "agent id should not be empty") + require.NoError(t, cb(ctx, id), "callback for each agent id failed") + } + } +} + // k8sStepRunInnerTests invokes the k8s inner tests inside the pods returned by the selector. Note that this // step requires the agent image to be built with the testing framework as there is the point where the binary // for the inner tests is copied @@ -1231,6 +1562,24 @@ func k8sStepRunInnerTests(agentPodLabelSelector string, expectedPodNumber int, c } } +// k8sStepHelmUninstall uninstalls the helm chart with the given release name +func k8sStepHelmUninstall(releaseName string) k8sTestStep { + return func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) { + settings := cli.New() + settings.SetNamespace(namespace) + actionConfig := &action.Configuration{} + + err := actionConfig.Init(settings.RESTClientGetter(), settings.Namespace(), "", + func(format string, v ...interface{}) {}) + require.NoError(t, err, "failed to init helm action config") + + uninstallAction := action.NewUninstall(actionConfig) + uninstallAction.Wait = true + _, err = uninstallAction.Run(releaseName) + require.NoError(t, err, "failed to uninstall helm chart") + } +} + // k8sStepHelmDeploy deploys a helm chart with the given values and the release name func k8sStepHelmDeploy(chartPath string, releaseName string, values map[string]any) k8sTestStep { return func(t *testing.T, ctx context.Context, kCtx k8sContext, namespace string) { @@ -1254,10 +1603,7 @@ func k8sStepHelmDeploy(chartPath string, releaseName string, values map[string]a uninstallAction := action.NewUninstall(actionConfig) uninstallAction.Wait = true - _, err = uninstallAction.Run(releaseName) - if err != nil { - t.Logf("failed to uninstall helm chart: %v", err) - } + _, _ = uninstallAction.Run(releaseName) }) installAction := action.NewInstall(actionConfig) @@ -1375,3 +1721,34 @@ func k8sStepCheckRestrictUpgrade(agentPodLabelSelector string, expectedPodNumber } } } + +// GetAgentResponse extends kibana.GetAgentResponse and includes the EnrolledAt field +type GetAgentResponse struct { + kibana.GetAgentResponse `json:",inline"` + EnrolledAt time.Time `json:"enrolled_at"` +} + +// kibanaGetAgent essentially re-implements kibana.GetAgent to extract also GetAgentResponse.EnrolledAt +func kibanaGetAgent(ctx context.Context, kc *kibana.Client, id string) (*GetAgentResponse, error) { + apiURL := fmt.Sprintf("/api/fleet/agents/%s", id) + r, err := kc.Connection.SendWithContext(ctx, http.MethodGet, apiURL, nil, nil, nil) + if err != nil { + return nil, fmt.Errorf("error calling get agent API: %w", err) + } + defer r.Body.Close() + var agentResp struct { + Item GetAgentResponse `json:"item"` + } + b, err := io.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("reading response body: %w", err) + } + if r.StatusCode != http.StatusOK { + return nil, fmt.Errorf("error calling get agent API: %s", string(b)) + } + err = json.Unmarshal(b, &agentResp) + if err != nil { + return nil, fmt.Errorf("unmarshalling response json: %w", err) + } + return &agentResp.Item, nil +} diff --git a/testing/mocks/internal_/pkg/agent/application/actions/handlers/diagnostics_provider_mock.go b/testing/mocks/internal_/pkg/agent/application/actions/handlers/diagnostics_provider_mock.go index 0004873c973..7a166506eaf 100644 --- a/testing/mocks/internal_/pkg/agent/application/actions/handlers/diagnostics_provider_mock.go +++ b/testing/mocks/internal_/pkg/agent/application/actions/handlers/diagnostics_provider_mock.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.51.1. DO NOT EDIT. package handlers @@ -33,7 +33,7 @@ func (_m *DiagnosticsProvider) EXPECT() *DiagnosticsProvider_Expecter { return &DiagnosticsProvider_Expecter{mock: &_m.Mock} } -// DiagnosticHooks provides a mock function with given fields: +// DiagnosticHooks provides a mock function with no fields func (_m *DiagnosticsProvider) DiagnosticHooks() diagnostics.Hooks { ret := _m.Called() diff --git a/testing/mocks/internal_/pkg/agent/application/actions/handlers/log_level_setter_mock.go b/testing/mocks/internal_/pkg/agent/application/actions/handlers/log_level_setter_mock.go index 1842d03ceec..c937315d8a0 100644 --- a/testing/mocks/internal_/pkg/agent/application/actions/handlers/log_level_setter_mock.go +++ b/testing/mocks/internal_/pkg/agent/application/actions/handlers/log_level_setter_mock.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.51.1. DO NOT EDIT. package handlers diff --git a/testing/mocks/internal_/pkg/agent/application/actions/handlers/uploader_mock.go b/testing/mocks/internal_/pkg/agent/application/actions/handlers/uploader_mock.go index e56e0ad0a88..f89f8b85641 100644 --- a/testing/mocks/internal_/pkg/agent/application/actions/handlers/uploader_mock.go +++ b/testing/mocks/internal_/pkg/agent/application/actions/handlers/uploader_mock.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.51.1. DO NOT EDIT. package handlers diff --git a/testing/mocks/internal_/pkg/agent/application/info/agent_mock.go b/testing/mocks/internal_/pkg/agent/application/info/agent_mock.go index ccec9b2929e..c61b0bab787 100644 --- a/testing/mocks/internal_/pkg/agent/application/info/agent_mock.go +++ b/testing/mocks/internal_/pkg/agent/application/info/agent_mock.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.51.1. DO NOT EDIT. package info @@ -25,7 +25,7 @@ func (_m *Agent) EXPECT() *Agent_Expecter { return &Agent_Expecter{mock: &_m.Mock} } -// AgentID provides a mock function with given fields: +// AgentID provides a mock function with no fields func (_m *Agent) AgentID() string { ret := _m.Called() @@ -70,7 +70,7 @@ func (_c *Agent_AgentID_Call) RunAndReturn(run func() string) *Agent_AgentID_Cal return _c } -// Headers provides a mock function with given fields: +// Headers provides a mock function with no fields func (_m *Agent) Headers() map[string]string { ret := _m.Called() @@ -117,7 +117,7 @@ func (_c *Agent_Headers_Call) RunAndReturn(run func() map[string]string) *Agent_ return _c } -// IsStandalone provides a mock function with given fields: +// IsStandalone provides a mock function with no fields func (_m *Agent) IsStandalone() bool { ret := _m.Called() @@ -162,7 +162,7 @@ func (_c *Agent_IsStandalone_Call) RunAndReturn(run func() bool) *Agent_IsStanda return _c } -// LogLevel provides a mock function with given fields: +// LogLevel provides a mock function with no fields func (_m *Agent) LogLevel() string { ret := _m.Called() @@ -207,7 +207,7 @@ func (_c *Agent_LogLevel_Call) RunAndReturn(run func() string) *Agent_LogLevel_C return _c } -// RawLogLevel provides a mock function with given fields: +// RawLogLevel provides a mock function with no fields func (_m *Agent) RawLogLevel() string { ret := _m.Called() @@ -345,7 +345,7 @@ func (_c *Agent_SetLogLevel_Call) RunAndReturn(run func(context.Context, string) return _c } -// Snapshot provides a mock function with given fields: +// Snapshot provides a mock function with no fields func (_m *Agent) Snapshot() bool { ret := _m.Called() @@ -390,7 +390,7 @@ func (_c *Agent_Snapshot_Call) RunAndReturn(run func() bool) *Agent_Snapshot_Cal return _c } -// Unprivileged provides a mock function with given fields: +// Unprivileged provides a mock function with no fields func (_m *Agent) Unprivileged() bool { ret := _m.Called() @@ -435,7 +435,7 @@ func (_c *Agent_Unprivileged_Call) RunAndReturn(run func() bool) *Agent_Unprivil return _c } -// Version provides a mock function with given fields: +// Version provides a mock function with no fields func (_m *Agent) Version() string { ret := _m.Called() diff --git a/testing/mocks/internal_/pkg/agent/storage/storage_mock.go b/testing/mocks/internal_/pkg/agent/storage/storage_mock.go new file mode 100644 index 00000000000..44f62085db7 --- /dev/null +++ b/testing/mocks/internal_/pkg/agent/storage/storage_mock.go @@ -0,0 +1,198 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +// Code generated by mockery v2.51.1. DO NOT EDIT. + +package storage + +import ( + io "io" + + mock "github.com/stretchr/testify/mock" +) + +// Storage is an autogenerated mock type for the Storage type +type Storage struct { + mock.Mock +} + +type Storage_Expecter struct { + mock *mock.Mock +} + +func (_m *Storage) EXPECT() *Storage_Expecter { + return &Storage_Expecter{mock: &_m.Mock} +} + +// Exists provides a mock function with no fields +func (_m *Storage) Exists() (bool, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Exists") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func() (bool, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Storage_Exists_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Exists' +type Storage_Exists_Call struct { + *mock.Call +} + +// Exists is a helper method to define mock.On call +func (_e *Storage_Expecter) Exists() *Storage_Exists_Call { + return &Storage_Exists_Call{Call: _e.mock.On("Exists")} +} + +func (_c *Storage_Exists_Call) Run(run func()) *Storage_Exists_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Storage_Exists_Call) Return(_a0 bool, _a1 error) *Storage_Exists_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Storage_Exists_Call) RunAndReturn(run func() (bool, error)) *Storage_Exists_Call { + _c.Call.Return(run) + return _c +} + +// Load provides a mock function with no fields +func (_m *Storage) Load() (io.ReadCloser, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Load") + } + + var r0 io.ReadCloser + var r1 error + if rf, ok := ret.Get(0).(func() (io.ReadCloser, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() io.ReadCloser); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(io.ReadCloser) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Storage_Load_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Load' +type Storage_Load_Call struct { + *mock.Call +} + +// Load is a helper method to define mock.On call +func (_e *Storage_Expecter) Load() *Storage_Load_Call { + return &Storage_Load_Call{Call: _e.mock.On("Load")} +} + +func (_c *Storage_Load_Call) Run(run func()) *Storage_Load_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Storage_Load_Call) Return(_a0 io.ReadCloser, _a1 error) *Storage_Load_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Storage_Load_Call) RunAndReturn(run func() (io.ReadCloser, error)) *Storage_Load_Call { + _c.Call.Return(run) + return _c +} + +// Save provides a mock function with given fields: _a0 +func (_m *Storage) Save(_a0 io.Reader) error { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for Save") + } + + var r0 error + if rf, ok := ret.Get(0).(func(io.Reader) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Storage_Save_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Save' +type Storage_Save_Call struct { + *mock.Call +} + +// Save is a helper method to define mock.On call +// - _a0 io.Reader +func (_e *Storage_Expecter) Save(_a0 interface{}) *Storage_Save_Call { + return &Storage_Save_Call{Call: _e.mock.On("Save", _a0)} +} + +func (_c *Storage_Save_Call) Run(run func(_a0 io.Reader)) *Storage_Save_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(io.Reader)) + }) + return _c +} + +func (_c *Storage_Save_Call) Return(_a0 error) *Storage_Save_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Storage_Save_Call) RunAndReturn(run func(io.Reader) error) *Storage_Save_Call { + _c.Call.Return(run) + return _c +} + +// NewStorage creates a new instance of Storage. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewStorage(t interface { + mock.TestingT + Cleanup(func()) +}) *Storage { + mock := &Storage{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/testing/mocks/internal_/pkg/fleetapi/acker/acker_mock.go b/testing/mocks/internal_/pkg/fleetapi/acker/acker_mock.go index d397b4b473c..79446571246 100644 --- a/testing/mocks/internal_/pkg/fleetapi/acker/acker_mock.go +++ b/testing/mocks/internal_/pkg/fleetapi/acker/acker_mock.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.51.1. DO NOT EDIT. package acker diff --git a/testing/mocks/internal_/pkg/fleetapi/client/sender_mock.go b/testing/mocks/internal_/pkg/fleetapi/client/sender_mock.go new file mode 100644 index 00000000000..ff40afa3ee1 --- /dev/null +++ b/testing/mocks/internal_/pkg/fleetapi/client/sender_mock.go @@ -0,0 +1,153 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +// Code generated by mockery v2.51.1. DO NOT EDIT. + +package client + +import ( + context "context" + http "net/http" + + io "io" + + mock "github.com/stretchr/testify/mock" + + url "net/url" +) + +// Sender is an autogenerated mock type for the Sender type +type Sender struct { + mock.Mock +} + +type Sender_Expecter struct { + mock *mock.Mock +} + +func (_m *Sender) EXPECT() *Sender_Expecter { + return &Sender_Expecter{mock: &_m.Mock} +} + +// Send provides a mock function with given fields: ctx, method, path, params, headers, body +func (_m *Sender) Send(ctx context.Context, method string, path string, params url.Values, headers http.Header, body io.Reader) (*http.Response, error) { + ret := _m.Called(ctx, method, path, params, headers, body) + + if len(ret) == 0 { + panic("no return value specified for Send") + } + + var r0 *http.Response + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, url.Values, http.Header, io.Reader) (*http.Response, error)); ok { + return rf(ctx, method, path, params, headers, body) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, url.Values, http.Header, io.Reader) *http.Response); ok { + r0 = rf(ctx, method, path, params, headers, body) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*http.Response) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, url.Values, http.Header, io.Reader) error); ok { + r1 = rf(ctx, method, path, params, headers, body) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Sender_Send_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Send' +type Sender_Send_Call struct { + *mock.Call +} + +// Send is a helper method to define mock.On call +// - ctx context.Context +// - method string +// - path string +// - params url.Values +// - headers http.Header +// - body io.Reader +func (_e *Sender_Expecter) Send(ctx interface{}, method interface{}, path interface{}, params interface{}, headers interface{}, body interface{}) *Sender_Send_Call { + return &Sender_Send_Call{Call: _e.mock.On("Send", ctx, method, path, params, headers, body)} +} + +func (_c *Sender_Send_Call) Run(run func(ctx context.Context, method string, path string, params url.Values, headers http.Header, body io.Reader)) *Sender_Send_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(url.Values), args[4].(http.Header), args[5].(io.Reader)) + }) + return _c +} + +func (_c *Sender_Send_Call) Return(_a0 *http.Response, _a1 error) *Sender_Send_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Sender_Send_Call) RunAndReturn(run func(context.Context, string, string, url.Values, http.Header, io.Reader) (*http.Response, error)) *Sender_Send_Call { + _c.Call.Return(run) + return _c +} + +// URI provides a mock function with no fields +func (_m *Sender) URI() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for URI") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// Sender_URI_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'URI' +type Sender_URI_Call struct { + *mock.Call +} + +// URI is a helper method to define mock.On call +func (_e *Sender_Expecter) URI() *Sender_URI_Call { + return &Sender_URI_Call{Call: _e.mock.On("URI")} +} + +func (_c *Sender_URI_Call) Run(run func()) *Sender_URI_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Sender_URI_Call) Return(_a0 string) *Sender_URI_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Sender_URI_Call) RunAndReturn(run func() string) *Sender_URI_Call { + _c.Call.Return(run) + return _c +} + +// NewSender creates a new instance of Sender. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewSender(t interface { + mock.TestingT + Cleanup(func()) +}) *Sender { + mock := &Sender{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/testing/mocks/pkg/control/v2/client/client_mock.go b/testing/mocks/pkg/control/v2/client/client_mock.go index 203a2411d6a..281086ad6f3 100644 --- a/testing/mocks/pkg/control/v2/client/client_mock.go +++ b/testing/mocks/pkg/control/v2/client/client_mock.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -// Code generated by mockery v2.42.2. DO NOT EDIT. +// Code generated by mockery v2.51.1. DO NOT EDIT. package client @@ -345,7 +345,7 @@ func (_c *Client_DiagnosticUnits_Call) RunAndReturn(run func(context.Context, .. return _c } -// Disconnect provides a mock function with given fields: +// Disconnect provides a mock function with no fields func (_m *Client) Disconnect() { _m.Called() } @@ -373,7 +373,7 @@ func (_c *Client_Disconnect_Call) Return() *Client_Disconnect_Call { } func (_c *Client_Disconnect_Call) RunAndReturn(run func()) *Client_Disconnect_Call { - _c.Call.Return(run) + _c.Run(run) return _c }