From d5cd46b727d58f79992a94aebf059e0650b3e668 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Fri, 20 Sep 2024 11:55:47 -0300 Subject: [PATCH] Make software batch endpoint asynchronous (#22258) #22069 API changes: https://github.com/fleetdm/fleet/pull/22259 QAd by applying 10 pieces of software on a team, which took 3+ minutes in total (which, before these changes was timing out at 100s.) With this approach, a GitOps CI run timing out might leave the background process running (which will eventually be applied to the database). The team discussed and agreed that we can fix this edge case later. - [X] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files) for more information. - [X] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements) - [X] Added/updated tests - [X] Manual QA for all new/changed functionality --- changes/22069-gitops-async-software-batch | 1 + cmd/fleet/serve.go | 2 + cmd/fleetctl/get_test.go | 4 +- cmd/fleetctl/gitops_test.go | 82 ++++- ee/server/service/mdm_external_test.go | 1 + ee/server/service/service.go | 3 + ee/server/service/software_installers.go | 281 +++++++++++++----- server/datastore/mysql/software_installers.go | 39 +-- .../mysql/software_installers_test.go | 24 +- server/fleet/datastore.go | 3 +- server/fleet/service.go | 26 +- server/mock/datastore_mock.go | 16 +- server/service/client_software.go | 33 +- server/service/client_teams.go | 7 +- server/service/handler.go | 3 + server/service/integration_enterprise_test.go | 179 +++++++++-- .../redis_key_value/redis_key_value.go | 58 ++++ .../redis_key_value/redis_key_value_test.go | 92 ++++++ server/service/software_installers.go | 49 ++- server/service/testing_utils.go | 9 + 20 files changed, 757 insertions(+), 155 deletions(-) create mode 100644 changes/22069-gitops-async-software-batch create mode 100644 server/service/redis_key_value/redis_key_value.go create mode 100644 server/service/redis_key_value/redis_key_value_test.go diff --git a/changes/22069-gitops-async-software-batch b/changes/22069-gitops-async-software-batch new file mode 100644 index 000000000000..35f0652fe209 --- /dev/null +++ b/changes/22069-gitops-async-software-batch @@ -0,0 +1 @@ +* Modified `POST /api/latest/fleet/software/batch` endpoint to be asynchronous and added a new endpoint `GET /api/latest/fleet/software/batch/{request_uuid}` to retrieve the result of the batch upload. diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 467713577ea8..e330cedcb34b 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -49,6 +49,7 @@ import ( "github.com/fleetdm/fleet/v4/server/pubsub" "github.com/fleetdm/fleet/v4/server/service" "github.com/fleetdm/fleet/v4/server/service/async" + "github.com/fleetdm/fleet/v4/server/service/redis_key_value" "github.com/fleetdm/fleet/v4/server/service/redis_lock" "github.com/fleetdm/fleet/v4/server/service/redis_policy_set" "github.com/fleetdm/fleet/v4/server/sso" @@ -798,6 +799,7 @@ the way that the Fleet server works. softwareInstallStore, bootstrapPackageStore, distributedLock, + redis_key_value.New(redisPool), ) if err != nil { initFatal(err, "initial Fleet Premium service") diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index e775f4ea5fb4..f39ff1cd5551 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -2320,8 +2320,8 @@ func TestGetTeamsYAMLAndApply(t *testing.T) { declaration.DeclarationUUID = uuid.NewString() return declaration, nil } - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { - return nil, nil + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil } actualYaml := runAppForTest(t, []string{"get", "teams", "--yaml"}) diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go index 64cb9fda19cc..b93496155999 100644 --- a/cmd/fleetctl/gitops_test.go +++ b/cmd/fleetctl/gitops_test.go @@ -182,7 +182,8 @@ func TestGitOpsBasicGlobalPremium(t *testing.T) { license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)} _, ds := runServerWithMockedDS( t, &service.TestServerOpts{ - License: license, + License: license, + KeyValueStore: newMemKeyValueStore(), }, ) @@ -229,7 +230,10 @@ func TestGitOpsBasicGlobalPremium(t *testing.T) { ds.NewJobFunc = func(ctx context.Context, job *fleet.Job) (*fleet.Job, error) { return &fleet.Job{}, nil } - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } @@ -285,7 +289,8 @@ func TestGitOpsBasicTeam(t *testing.T) { license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)} _, ds := runServerWithMockedDS( t, &service.TestServerOpts{ - License: license, + License: license, + KeyValueStore: newMemKeyValueStore(), }, ) @@ -373,7 +378,10 @@ func TestGitOpsBasicTeam(t *testing.T) { ds.DeleteMDMAppleDeclarationByNameFunc = func(ctx context.Context, teamID *uint, name string) error { return nil } - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } ds.ApplyEnrollSecretsFunc = func(ctx context.Context, teamID *uint, secrets []*fleet.EnrollSecret) error { @@ -644,6 +652,7 @@ func TestGitOpsFullTeam(t *testing.T) { MDMPusher: mockPusher{}, FleetConfig: &fleetCfg, NoCacheDatastore: true, + KeyValueStore: newMemKeyValueStore(), }, ) @@ -804,8 +813,11 @@ func TestGitOpsFullTeam(t *testing.T) { return nil } var appliedSoftwareInstallers []*fleet.UploadSoftwareInstallerPayload - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { appliedSoftwareInstallers = installers + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } ds.SetTeamVPPAppsFunc = func(ctx context.Context, teamID *uint, adamIDs []fleet.VPPAppTeam) error { @@ -937,7 +949,8 @@ func TestGitOpsBasicGlobalAndTeam(t *testing.T) { license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)} _, ds := runServerWithMockedDS( t, &service.TestServerOpts{ - License: license, + License: license, + KeyValueStore: newMemKeyValueStore(), }, ) @@ -1055,7 +1068,10 @@ func TestGitOpsBasicGlobalAndTeam(t *testing.T) { savedTeam = team return team, nil } - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } ds.ListSoftwareTitlesFunc = func(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter) ([]fleet.SoftwareTitleListResult, int, *fleet.PaginationMetadata, error) { @@ -1201,7 +1217,8 @@ func TestGitOpsBasicGlobalAndNoTeam(t *testing.T) { license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)} _, ds := runServerWithMockedDS( t, &service.TestServerOpts{ - License: license, + License: license, + KeyValueStore: newMemKeyValueStore(), }, ) // Mock appConfig @@ -1317,7 +1334,10 @@ func TestGitOpsBasicGlobalAndNoTeam(t *testing.T) { savedTeam = team return team, nil } - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } ds.ListSoftwareTitlesFunc = func(ctx context.Context, opt fleet.SoftwareTitleListOptions, tmFilter fleet.TeamFilter) ([]fleet.SoftwareTitleListResult, int, *fleet.PaginationMetadata, error) { @@ -1634,9 +1654,9 @@ func TestGitOpsTeamSofwareInstallers(t *testing.T) { file string wantErr string }{ - {"testdata/gitops/team_software_installer_not_found.yml", "Please make sure that URLs are publicy accessible to the internet."}, + {"testdata/gitops/team_software_installer_not_found.yml", "Please make sure that URLs are reachable from your Fleet server."}, {"testdata/gitops/team_software_installer_unsupported.yml", "The file should be .pkg, .msi, .exe or .deb."}, - {"testdata/gitops/team_software_installer_too_large.yml", "The maximum file size is 500 MB"}, + {"testdata/gitops/team_software_installer_too_large.yml", "The maximum file size is 500 MiB"}, {"testdata/gitops/team_software_installer_valid.yml", ""}, {"testdata/gitops/team_software_installer_valid_apply.yml", ""}, {"testdata/gitops/team_software_installer_pre_condition_multiple_queries.yml", "should have only one query."}, @@ -1668,10 +1688,13 @@ func TestGitOpsTeamSoftwareInstallersQueryEnv(t *testing.T) { t.Setenv("QUERY_VAR", "IT_WORKS") - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { if installers[0].PreInstallQuery != "select IT_WORKS" { - return nil, fmt.Errorf("Missing env var, got %s", installers[0].PreInstallQuery) + return fmt.Errorf("Missing env var, got %s", installers[0].PreInstallQuery) } + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } @@ -1686,9 +1709,9 @@ func TestGitOpsNoTeamSoftwareInstallers(t *testing.T) { noTeamFile string wantErr string }{ - {"testdata/gitops/no_team_software_installer_not_found.yml", "Please make sure that URLs are publicy accessible to the internet."}, + {"testdata/gitops/no_team_software_installer_not_found.yml", "Please make sure that URLs are reachable from your Fleet server."}, {"testdata/gitops/no_team_software_installer_unsupported.yml", "The file should be .pkg, .msi, .exe or .deb."}, - {"testdata/gitops/no_team_software_installer_too_large.yml", "The maximum file size is 500 MB"}, + {"testdata/gitops/no_team_software_installer_too_large.yml", "The maximum file size is 500 MiB"}, {"testdata/gitops/no_team_software_installer_valid.yml", ""}, {"testdata/gitops/no_team_software_installer_pre_condition_multiple_queries.yml", "should have only one query."}, {"testdata/gitops/no_team_software_installer_pre_condition_not_found.yml", "no such file or directory"}, @@ -2050,6 +2073,7 @@ func setupFullGitOpsPremiumServer(t *testing.T) (*mock.Store, **fleet.AppConfig, FleetConfig: &fleetCfg, License: license, NoCacheDatastore: true, + KeyValueStore: newMemKeyValueStore(), }, ) @@ -2181,7 +2205,10 @@ func setupFullGitOpsPremiumServer(t *testing.T) (*mock.Store, **fleet.AppConfig, declaration.DeclarationUUID = uuid.NewString() return declaration, nil } - ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { + ds.BatchSetSoftwareInstallersFunc = func(ctx context.Context, teamID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { + return nil + } + ds.GetSoftwareInstallersFunc = func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { return nil, nil } @@ -2890,3 +2917,26 @@ software: }) } } + +type memKeyValueStore struct { + m map[string]string +} + +func newMemKeyValueStore() *memKeyValueStore { + return &memKeyValueStore{ + m: make(map[string]string), + } +} + +func (m *memKeyValueStore) Set(ctx context.Context, key string, value string, expireTime time.Duration) error { + m.m[key] = value + return nil +} + +func (m *memKeyValueStore) Get(ctx context.Context, key string) (*string, error) { + v, ok := m.m[key] + if !ok { + return nil, nil + } + return &v, nil +} diff --git a/ee/server/service/mdm_external_test.go b/ee/server/service/mdm_external_test.go index 1d92d1ce3b43..760d046c9c5b 100644 --- a/ee/server/service/mdm_external_test.go +++ b/ee/server/service/mdm_external_test.go @@ -109,6 +109,7 @@ func setupMockDatastorePremiumService(t testing.TB) (*mock.Store, *eeservice.Ser nil, nil, nil, + nil, ) if err != nil { panic(err) diff --git a/ee/server/service/service.go b/ee/server/service/service.go index 7ef6f8b8a530..fb66f21136ad 100644 --- a/ee/server/service/service.go +++ b/ee/server/service/service.go @@ -30,6 +30,7 @@ type Service struct { softwareInstallStore fleet.SoftwareInstallerStore bootstrapPackageStore fleet.MDMBootstrapPackageStore distributedLock fleet.Lock + keyValueStore fleet.KeyValueStore } func NewService( @@ -46,6 +47,7 @@ func NewService( softwareInstallStore fleet.SoftwareInstallerStore, bootstrapPackageStore fleet.MDMBootstrapPackageStore, distributedLock fleet.Lock, + keyValueStore fleet.KeyValueStore, ) (*Service, error) { authorizer, err := authz.NewAuthorizer() if err != nil { @@ -67,6 +69,7 @@ func NewService( softwareInstallStore: softwareInstallStore, bootstrapPackageStore: bootstrapPackageStore, distributedLock: distributedLock, + keyValueStore: keyValueStore, } // Override methods that can't be easily overriden via diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index 5a1d67910665..ac4461a592b2 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -14,6 +14,7 @@ import ( "path/filepath" "regexp" "strings" + "time" "github.com/fleetdm/fleet/v4/pkg/file" "github.com/fleetdm/fleet/v4/pkg/fleethttp" @@ -24,6 +25,7 @@ import ( "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mdm/apple/vpp" "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/go-kit/log" kitlog "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/google/uuid" @@ -1112,13 +1114,21 @@ func (svc *Service) addMetadataToSoftwarePayload(ctx context.Context, payload *f return meta.Extension, nil } -const maxInstallerSizeBytes int64 = 1024 * 1024 * 500 +const ( + maxInstallerSizeBytes int64 = 1024 * 1024 * 500 + batchSoftwarePrefix = "software_batch_" +) func (svc *Service) BatchSetSoftwareInstallers( ctx context.Context, tmName string, payloads []fleet.SoftwareInstallerPayload, dryRun bool, -) ([]fleet.SoftwarePackageResponse, error) { +) (string, error) { if err := svc.authz.Authorize(ctx, &fleet.Team{}, fleet.ActionRead); err != nil { - return nil, err + return "", err + } + + vc, ok := viewer.FromContext(ctx) + if !ok { + return "", fleet.ErrNoContext } var teamID *uint @@ -1127,98 +1137,165 @@ func (svc *Service) BatchSetSoftwareInstallers( if err != nil { // If this is a dry run, the team may not have been created yet if dryRun && fleet.IsNotFound(err) { - return nil, nil + return "", nil } - return nil, err + return "", err } teamID = &tm.ID } if err := svc.authz.Authorize(ctx, &fleet.SoftwareInstaller{TeamID: teamID}, fleet.ActionWrite); err != nil { - return nil, ctxerr.Wrap(ctx, err, "validating authorization") + return "", ctxerr.Wrap(ctx, err, "validating authorization") } + // Verify payloads first, to prevent starting the download+upload process if the data is invalid. for _, payload := range payloads { if len(payload.URL) > fleet.SoftwareInstallerURLMaxLength { - return nil, fleet.NewInvalidArgumentError( + return "", fleet.NewInvalidArgumentError( "software.url", "software URL is too long, must be less than 256 characters", ) } + if _, err := url.ParseRequestURI(payload.URL); err != nil { + return "", fleet.NewInvalidArgumentError( + "software.url", + fmt.Sprintf("Couldn't edit software. URL (%q) is invalid", payload.URL), + ) + } } - vc, ok := viewer.FromContext(ctx) - if !ok { - return nil, fleet.ErrNoContext + // keyExpireTime is the current maximum time supported for retrieving + // the result of a software by batch operation. + const keyExpireTime = 24 * time.Hour + + requestUUID := uuid.NewString() + if err := svc.keyValueStore.Set(ctx, batchSoftwarePrefix+requestUUID, batchSetProcessing, keyExpireTime); err != nil { + return "", ctxerr.Wrapf(ctx, err, "failed to set key as %s", batchSetProcessing) } - g, workerCtx := errgroup.WithContext(ctx) - g.SetLimit(3) - // critical to avoid data race, the slice is pre-allocated and each - // goroutine only writes to its index. - installers := make([]*fleet.UploadSoftwareInstallerPayload, len(payloads)) + svc.logger.Log( + "msg", "software batch start", + "request_uuid", requestUUID, + "team_id", teamID, + "payloads", len(payloads), + ) - for i, p := range payloads { - i, p := i, p + go svc.softwareBatchUpload( + requestUUID, + teamID, + vc.UserID(), + payloads, + dryRun, + ) - g.Go(func() error { - // validate the URL before doing the request - _, err := url.ParseRequestURI(p.URL) - if err != nil { - return fleet.NewInvalidArgumentError( + return requestUUID, nil +} + +const ( + batchSetProcessing = "processing" + batchSetCompleted = "completed" + batchSetFailedPrefix = "failed:" +) + +func (svc *Service) softwareBatchUpload( + requestUUID string, + teamID *uint, + userID uint, + payloads []fleet.SoftwareInstallerPayload, + dryRun bool, +) { + var batchErr error + + // We do not use the request ctx on purpose because this method runs in the background. + ctx := context.Background() + + defer func(start time.Time) { + status := batchSetCompleted + if batchErr != nil { + status = fmt.Sprintf("%s%s", batchSetFailedPrefix, batchErr) + } + logger := log.With(svc.logger, + "request_uuid", requestUUID, + "team_id", teamID, + "payloads", len(payloads), + "status", status, + "took", time.Since(start), + ) + logger.Log("msg", "software batch done") + // Give 10m for the client to read the result (it overrides the previos expiration time). + if err := svc.keyValueStore.Set(ctx, batchSoftwarePrefix+requestUUID, status, 10*time.Minute); err != nil { + logger.Log("msg", "failed to set result", "err", err) + } + }(time.Now()) + + downloadURLFn := func(ctx context.Context, url string) (http.Header, []byte, error) { + client := fleethttp.NewClient() + client.Transport = fleethttp.NewSizeLimitTransport(maxInstallerSizeBytes) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, nil, fmt.Errorf("creating request for URL %q: %w", url, err) + } + + resp, err := client.Do(req) + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.Is(err, fleethttp.ErrMaxSizeExceeded) || errors.As(err, &maxBytesErr) { + return nil, nil, fleet.NewInvalidArgumentError( "software.url", - fmt.Sprintf("Couldn't edit software. URL (%q) is invalid", p.URL), + fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d MiB", url, maxInstallerSizeBytes/(1024*1024)), ) } - client := fleethttp.NewClient() - client.Transport = fleethttp.NewSizeLimitTransport(maxInstallerSizeBytes) - req, err := http.NewRequestWithContext(workerCtx, http.MethodGet, p.URL, nil) - if err != nil { - return ctxerr.Wrapf(ctx, err, "creating request for URL %s", p.URL) - } + return nil, nil, fmt.Errorf("performing request for URL %q: %w", url, err) + } + defer resp.Body.Close() - resp, err := client.Do(req) - if err != nil { - var maxBytesErr *http.MaxBytesError - if errors.Is(err, fleethttp.ErrMaxSizeExceeded) || errors.As(err, &maxBytesErr) { - return fleet.NewInvalidArgumentError( - "software.url", - fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d MB", p.URL, maxInstallerSizeBytes/(1024*1024)), - ) - } + if resp.StatusCode == http.StatusNotFound { + return nil, nil, fleet.NewInvalidArgumentError( + "software.url", + fmt.Sprintf("Couldn't edit software. URL (%q) returned \"Not Found\". Please make sure that URLs are reachable from your Fleet server.", url), + ) + } - return ctxerr.Wrapf(ctx, err, "performing request for URL %s", p.URL) - } - defer resp.Body.Close() + // Allow all 2xx and 3xx status codes in this pass. + if resp.StatusCode >= 400 { + return nil, nil, fleet.NewInvalidArgumentError( + "software.url", + fmt.Sprintf("Couldn't edit software. URL (%q) received response status code %d.", url, resp.StatusCode), + ) + } - if resp.StatusCode == http.StatusNotFound { - return fleet.NewInvalidArgumentError( + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + // the max size error can be received either at client.Do or here when + // reading the body if it's caught via a limited body reader. + var maxBytesErr *http.MaxBytesError + if errors.Is(err, fleethttp.ErrMaxSizeExceeded) || errors.As(err, &maxBytesErr) { + return nil, nil, fleet.NewInvalidArgumentError( "software.url", - fmt.Sprintf("Couldn't edit software. URL (%q) doesn't exist. Please make sure that URLs are publicy accessible to the internet.", p.URL), + fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d MiB", url, maxInstallerSizeBytes/(1024*1024)), ) } + return nil, nil, fmt.Errorf("reading installer %q contents: %w", url, err) + } - // Allow all 2xx and 3xx status codes in this pass. - if resp.StatusCode > 400 { - return fleet.NewInvalidArgumentError( - "software.url", - fmt.Sprintf("Couldn't edit software. URL (%q) received response status code %d.", p.URL, resp.StatusCode), - ) - } + return resp.Header, bodyBytes, nil + } + + var g errgroup.Group + g.SetLimit(3) + // critical to avoid data race, the slice is pre-allocated and each + // goroutine only writes to its index. + installers := make([]*fleet.UploadSoftwareInstallerPayload, len(payloads)) + + for i, p := range payloads { + i, p := i, p - bodyBytes, err := io.ReadAll(resp.Body) + g.Go(func() error { + headers, bodyBytes, err := downloadURLFn(ctx, p.URL) if err != nil { - // the max size error can be received either at client.Do or here when - // reading the body if it's caught via a limited body reader. - var maxBytesErr *http.MaxBytesError - if errors.Is(err, fleethttp.ErrMaxSizeExceeded) || errors.As(err, &maxBytesErr) { - return fleet.NewInvalidArgumentError( - "software.url", - fmt.Sprintf("Couldn't edit software. URL (%q). The maximum file size is %d MB", p.URL, maxInstallerSizeBytes/(1024*1024)), - ) - } - return ctxerr.Wrapf(ctx, err, "reading installer %q contents", p.URL) + return err } installer := &fleet.UploadSoftwareInstallerPayload{ @@ -1229,13 +1306,13 @@ func (svc *Service) BatchSetSoftwareInstallers( UninstallScript: p.UninstallScript, InstallerFile: bytes.NewReader(bodyBytes), SelfService: p.SelfService, - UserID: vc.UserID(), + UserID: userID, URL: p.URL, } // set the filename before adding metadata, as it is used as fallback var filename string - cdh, ok := resp.Header["Content-Disposition"] + cdh, ok := headers["Content-Disposition"] if ok && len(cdh) > 0 { _, params, err := mime.ParseMediaType(cdh[0]) if err == nil { @@ -1273,30 +1350,88 @@ func (svc *Service) BatchSetSoftwareInstallers( } if err := g.Wait(); err != nil { - // NOTE: intentionally not wrapping to avoid polluting user - // errors. - return nil, err + // NOTE: intentionally not wrapping to avoid polluting user errors. + batchErr = err + return } if dryRun { - return nil, nil + return } for _, payload := range installers { if err := svc.storeSoftware(ctx, payload); err != nil { - return nil, ctxerr.Wrap(ctx, err, "storing software installer") + batchErr = fmt.Errorf("storing software installer %q: %w", payload.Filename, err) + return } } - insertedSoftwareInstallers, err := svc.ds.BatchSetSoftwareInstallers(ctx, teamID, installers) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "batch set software installers") + if err := svc.ds.BatchSetSoftwareInstallers(ctx, teamID, installers); err != nil { + batchErr = fmt.Errorf("batch set software installers: %w", err) + return } // Note: per @noahtalerman we don't want activity items for CLI actions // anymore, so that's intentionally skipped. +} + +func (svc *Service) GetBatchSetSoftwareInstallersResult(ctx context.Context, tmName string, requestUUID string, dryRun bool) (string, string, []fleet.SoftwarePackageResponse, error) { + // We've already authorized in the POST /api/latest/fleet/software/batch, + // but adding it here so we don't need to worry about a special case endpoint. + if err := svc.authz.Authorize(ctx, &fleet.Team{}, fleet.ActionRead); err != nil { + return "", "", nil, err + } + + result, err := svc.keyValueStore.Get(ctx, batchSoftwarePrefix+requestUUID) + if err != nil { + return "", "", nil, ctxerr.Wrap(ctx, err, "failed to get result") + } + if result == nil { + return "", "", nil, ctxerr.Wrap(ctx, notFoundError{}, "request_uuid not found") + } + + switch { + case *result == batchSetCompleted: + if dryRun { + return fleet.BatchSetSoftwareInstallersStatusCompleted, "", nil, nil + } // this will fall through to retrieving software packages if not a dry run. + case *result == batchSetProcessing: + return fleet.BatchSetSoftwareInstallersStatusProcessing, "", nil, nil + case strings.HasPrefix(*result, batchSetFailedPrefix): + message := strings.TrimPrefix(*result, batchSetFailedPrefix) + return fleet.BatchSetSoftwareInstallersStatusFailed, message, nil, nil + default: + return "", "", nil, ctxerr.New(ctx, "invalid status") + } + + var ( + teamID uint // GetSoftwareInstallers uses 0 for "No team" + ptrTeamID *uint // Authorize uses *uint for "No team" teamID + ) + if tmName != "" { + team, err := svc.ds.TeamByName(ctx, tmName) + if err != nil { + return "", "", nil, ctxerr.Wrap(ctx, err, "load team by name") + } + teamID = team.ID + ptrTeamID = &team.ID + } + + // We've already authorized in the POST /api/latest/fleet/software/batch, + // but adding it here so we don't need to worry about a special case endpoint. + // + // We use fleet.ActionWrite because this method is the counterpart of the POST + // /api/latest/fleet/software/batch. + if err := svc.authz.Authorize(ctx, &fleet.SoftwareInstaller{TeamID: ptrTeamID}, fleet.ActionWrite); err != nil { + return "", "", nil, ctxerr.Wrap(ctx, err, "validating authorization") + } + + softwarePackages, err := svc.ds.GetSoftwareInstallers(ctx, teamID) + if err != nil { + return "", "", nil, ctxerr.Wrap(ctx, err, "get software installers") + } - return insertedSoftwareInstallers, nil + return fleet.BatchSetSoftwareInstallersStatusCompleted, "", softwarePackages, nil } func (svc *Service) SelfServiceInstallSoftwareTitle(ctx context.Context, host *fleet.Host, softwareTitleID uint) error { diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 5aa7a2f11d51..7d7f0169e3ad 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -768,7 +768,7 @@ func (ds *Datastore) CleanupUnusedSoftwareInstallers(ctx context.Context, softwa return ctxerr.Wrap(ctx, err, "cleanup unused software installers") } -func (ds *Datastore) BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { +func (ds *Datastore) BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { const upsertSoftwareTitles = ` INSERT INTO software_titles (name, source, browser) @@ -878,23 +878,12 @@ ON DUPLICATE KEY UPDATE url = VALUES(url) ` - const loadInsertedSoftwareInstallers = ` -SELECT - team_id, - title_id, - url -FROM - software_installers -WHERE global_or_team_id = ? -` - // use a team id of 0 if no-team var globalOrTeamID uint if tmID != nil { globalOrTeamID = *tmID } - var insertedSoftwareInstallers []fleet.SoftwarePackageResponse if err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { // if no installers are provided, just delete whatever was in // the table @@ -1040,15 +1029,11 @@ WHERE global_or_team_id = ? } } - if err := sqlx.SelectContext(ctx, tx, &insertedSoftwareInstallers, loadInsertedSoftwareInstallers, globalOrTeamID); err != nil { - return ctxerr.Wrap(ctx, err, "load inserted software installers") - } - return nil }); err != nil { - return nil, err + return err } - return insertedSoftwareInstallers, nil + return nil } func (ds *Datastore) HasSelfServiceSoftwareInstallers(ctx context.Context, hostPlatform string, hostTeamID *uint) (bool, error) { @@ -1135,3 +1120,21 @@ func (ds *Datastore) UpdateSoftwareInstallerWithoutPackageIDs(ctx context.Contex } return nil } + +func (ds *Datastore) GetSoftwareInstallers(ctx context.Context, teamID uint) ([]fleet.SoftwarePackageResponse, error) { + const loadInsertedSoftwareInstallers = ` +SELECT + team_id, + title_id, + url +FROM + software_installers +WHERE global_or_team_id = ? +` + var softwarePackages []fleet.SoftwarePackageResponse + // Using ds.writer(ctx) on purpose because this method is to be called after applying software. + if err := sqlx.SelectContext(ctx, ds.writer(ctx), &softwarePackages, loadInsertedSoftwareInstallers, teamID); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get software installers") + } + return softwarePackages, nil +} diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 862d70063a07..178b85807148 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -630,11 +630,15 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { } // batch set with everything empty - softwareInstallers, err := ds.BatchSetSoftwareInstallers(ctx, &team.ID, nil) + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, nil) + require.NoError(t, err) + softwareInstallers, err := ds.GetSoftwareInstallers(ctx, team.ID) require.NoError(t, err) require.Empty(t, softwareInstallers) assertSoftware(nil) - softwareInstallers, err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{}) + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{}) + require.NoError(t, err) + softwareInstallers, err = ds.GetSoftwareInstallers(ctx, team.ID) require.NoError(t, err) require.Empty(t, softwareInstallers) assertSoftware(nil) @@ -642,7 +646,7 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { // add a single installer ins0 := "installer0" ins0File := bytes.NewReader([]byte("installer0")) - softwareInstallers, err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{{ + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{{ InstallScript: "install", InstallerFile: ins0File, StorageID: ins0, @@ -656,6 +660,8 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { URL: "https://example.com", }}) require.NoError(t, err) + softwareInstallers, err = ds.GetSoftwareInstallers(ctx, team.ID) + require.NoError(t, err) require.Len(t, softwareInstallers, 1) require.NotNil(t, softwareInstallers[0].TeamID) require.Equal(t, team.ID, *softwareInstallers[0].TeamID) @@ -668,7 +674,7 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { // add a new installer + ins0 installer ins1 := "installer1" ins1File := bytes.NewReader([]byte("installer1")) - softwareInstallers, err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{ + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{ { InstallScript: "install", InstallerFile: ins0File, @@ -698,6 +704,8 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { }, }) require.NoError(t, err) + softwareInstallers, err = ds.GetSoftwareInstallers(ctx, team.ID) + require.NoError(t, err) require.Len(t, softwareInstallers, 2) require.NotNil(t, softwareInstallers[0].TitleID) require.NotNil(t, softwareInstallers[0].TeamID) @@ -713,7 +721,7 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { }) // remove ins0 - softwareInstallers, err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{ + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{ { InstallScript: "install", PostInstallScript: "post-install", @@ -728,6 +736,8 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { }, }) require.NoError(t, err) + softwareInstallers, err = ds.GetSoftwareInstallers(ctx, team.ID) + require.NoError(t, err) require.Len(t, softwareInstallers, 1) require.NotNil(t, softwareInstallers[0].TitleID) require.NotNil(t, softwareInstallers[0].TeamID) @@ -737,7 +747,9 @@ func testBatchSetSoftwareInstallers(t *testing.T, ds *Datastore) { }) // remove everything - softwareInstallers, err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{}) + err = ds.BatchSetSoftwareInstallers(ctx, &team.ID, []*fleet.UploadSoftwareInstallerPayload{}) + require.NoError(t, err) + softwareInstallers, err = ds.GetSoftwareInstallers(ctx, team.ID) require.NoError(t, err) require.Empty(t, softwareInstallers) assertSoftware([]fleet.SoftwareTitle{}) diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 9f9a9de50479..99b2cdb7d27c 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1711,7 +1711,8 @@ type Datastore interface { CleanupUnusedSoftwareInstallers(ctx context.Context, softwareInstallStore SoftwareInstallerStore, removeCreatedBefore time.Time) error // BatchSetSoftwareInstallers sets the software installers for the given team or no team. - BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*UploadSoftwareInstallerPayload) ([]SoftwarePackageResponse, error) + BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*UploadSoftwareInstallerPayload) error + GetSoftwareInstallers(ctx context.Context, tmID uint) ([]SoftwarePackageResponse, error) // HasSelfServiceSoftwareInstallers returns true if self-service software installers are available for the team or globally. HasSelfServiceSoftwareInstallers(ctx context.Context, platform string, teamID *uint) (bool, error) diff --git a/server/fleet/service.go b/server/fleet/service.go index 8599e464e1b5..24756ebb6d80 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -643,9 +643,15 @@ type Service interface { // GetSoftwareInstallResults gets the results for a particular software install attempt. GetSoftwareInstallResults(ctx context.Context, installUUID string) (*HostSoftwareInstallerResult, error) - // BatchSetSoftwareInstallers replaces the software installers for a specified team. - // Returns the metadata of inserted software installers. - BatchSetSoftwareInstallers(ctx context.Context, tmName string, payloads []SoftwareInstallerPayload, dryRun bool) ([]SoftwarePackageResponse, error) + // BatchSetSoftwareInstallers asynchronously replaces the software installers for a specified team. + // Returns a request UUID that can be used to track an ongoing batch request (with GetBatchSetSoftwareInstallersResult). + BatchSetSoftwareInstallers(ctx context.Context, tmName string, payloads []SoftwareInstallerPayload, dryRun bool) (string, error) + // GetBatchSetSoftwareInstallersResult polls for the status of a batch-apply started by BatchSetSoftwareInstallers. + // Return values: + // - 'status': status of the batch-apply which can be "processing", "completed" or "failed". + // - 'message': which contains error information when the status is "failed". + // - 'packages': Contains the list of the applied software packages (when status is "completed"). This is always empty for a dry run. + GetBatchSetSoftwareInstallersResult(ctx context.Context, tmName string, requestUUID string, dryRun bool) (status string, message string, packages []SoftwarePackageResponse, err error) // SelfServiceInstallSoftwareTitle installs a software title // initiated by the user @@ -1120,3 +1126,17 @@ type Service interface { // CalendarWebhook handles incoming calendar callback requests. CalendarWebhook(ctx context.Context, eventUUID string, channelID string, resourceState string) error } + +type KeyValueStore interface { + Set(ctx context.Context, key string, value string, expireTime time.Duration) error + Get(ctx context.Context, key string) (*string, error) +} + +const ( + // BatchSetSoftwareInstallerStatusProcessing is the value returned for an ongoing BatchSetSoftwareInstallers operation. + BatchSetSoftwareInstallersStatusProcessing = "processing" + // BatchSetSoftwareInstallerStatusCompleted is the value returned for a completed BatchSetSoftwareInstallers operation. + BatchSetSoftwareInstallersStatusCompleted = "completed" + // BatchSetSoftwareInstallerStatusFailed is the value returned for a failed BatchSetSoftwareInstallers operation. + BatchSetSoftwareInstallersStatusFailed = "failed" +) diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index bc009c4ea30d..a592559bdf5a 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -1070,7 +1070,9 @@ type GetSoftwareInstallResultsFunc func(ctx context.Context, resultsUUID string) type CleanupUnusedSoftwareInstallersFunc func(ctx context.Context, softwareInstallStore fleet.SoftwareInstallerStore, removeCreatedBefore time.Time) error -type BatchSetSoftwareInstallersFunc func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) +type BatchSetSoftwareInstallersFunc func(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error + +type GetSoftwareInstallersFunc func(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) type HasSelfServiceSoftwareInstallersFunc func(ctx context.Context, platform string, teamID *uint) (bool, error) @@ -2667,6 +2669,9 @@ type DataStore struct { BatchSetSoftwareInstallersFunc BatchSetSoftwareInstallersFunc BatchSetSoftwareInstallersFuncInvoked bool + GetSoftwareInstallersFunc GetSoftwareInstallersFunc + GetSoftwareInstallersFuncInvoked bool + HasSelfServiceSoftwareInstallersFunc HasSelfServiceSoftwareInstallersFunc HasSelfServiceSoftwareInstallersFuncInvoked bool @@ -6369,13 +6374,20 @@ func (s *DataStore) CleanupUnusedSoftwareInstallers(ctx context.Context, softwar return s.CleanupUnusedSoftwareInstallersFunc(ctx, softwareInstallStore, removeCreatedBefore) } -func (s *DataStore) BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) ([]fleet.SoftwarePackageResponse, error) { +func (s *DataStore) BatchSetSoftwareInstallers(ctx context.Context, tmID *uint, installers []*fleet.UploadSoftwareInstallerPayload) error { s.mu.Lock() s.BatchSetSoftwareInstallersFuncInvoked = true s.mu.Unlock() return s.BatchSetSoftwareInstallersFunc(ctx, tmID, installers) } +func (s *DataStore) GetSoftwareInstallers(ctx context.Context, tmID uint) ([]fleet.SoftwarePackageResponse, error) { + s.mu.Lock() + s.GetSoftwareInstallersFuncInvoked = true + s.mu.Unlock() + return s.GetSoftwareInstallersFunc(ctx, tmID) +} + func (s *DataStore) HasSelfServiceSoftwareInstallers(ctx context.Context, platform string, teamID *uint) (bool, error) { s.mu.Lock() s.HasSelfServiceSoftwareInstallersFuncInvoked = true diff --git a/server/service/client_software.go b/server/service/client_software.go index 413e6dc7e9d8..60a0911093f4 100644 --- a/server/service/client_software.go +++ b/server/service/client_software.go @@ -1,7 +1,10 @@ package service import ( + "errors" + "fmt" "net/url" + "time" "github.com/fleetdm/fleet/v4/server/fleet" ) @@ -29,14 +32,38 @@ func (c *Client) ListSoftwareTitles(query string) ([]fleet.SoftwareTitleListResu } func (c *Client) ApplyNoTeamSoftwareInstallers(softwareInstallers []fleet.SoftwareInstallerPayload, opts fleet.ApplySpecOptions) ([]fleet.SoftwarePackageResponse, error) { - verb, path := "POST", "/api/latest/fleet/software/batch" query, err := url.ParseQuery(opts.RawQuery()) if err != nil { return nil, err } + return c.applySoftwareInstallers(softwareInstallers, query, opts.DryRun) +} + +func (c *Client) applySoftwareInstallers(softwareInstallers []fleet.SoftwareInstallerPayload, query url.Values, dryRun bool) ([]fleet.SoftwarePackageResponse, error) { + path := "/api/latest/fleet/software/batch" var resp batchSetSoftwareInstallersResponse - if err := c.authenticatedRequestWithQuery(map[string]interface{}{"software": softwareInstallers}, verb, path, &resp, query.Encode()); err != nil { + if err := c.authenticatedRequestWithQuery(map[string]interface{}{"software": softwareInstallers}, "POST", path, &resp, query.Encode()); err != nil { return nil, err } - return resp.Packages, nil + if dryRun && resp.RequestUUID == "" { + return nil, nil + } + + requestUUID := resp.RequestUUID + for { + var resp batchSetSoftwareInstallersResultResponse + if err := c.authenticatedRequestWithQuery(nil, "GET", path+"/"+requestUUID, &resp, query.Encode()); err != nil { + return nil, err + } + switch { + case resp.Status == fleet.BatchSetSoftwareInstallersStatusProcessing: + time.Sleep(5 * time.Second) + case resp.Status == fleet.BatchSetSoftwareInstallersStatusFailed: + return nil, errors.New(resp.Message) + case resp.Status == fleet.BatchSetSoftwareInstallersStatusCompleted: + return resp.Packages, nil + default: + return nil, fmt.Errorf("unknown status: %q", resp.Status) + } + } } diff --git a/server/service/client_teams.go b/server/service/client_teams.go index 5c5180a6b700..5d541e903c9b 100644 --- a/server/service/client_teams.go +++ b/server/service/client_teams.go @@ -94,17 +94,12 @@ func (c *Client) ApplyTeamScripts(tmName string, scripts []fleet.ScriptPayload, } func (c *Client) ApplyTeamSoftwareInstallers(tmName string, softwareInstallers []fleet.SoftwareInstallerPayload, opts fleet.ApplySpecOptions) ([]fleet.SoftwarePackageResponse, error) { - verb, path := "POST", "/api/latest/fleet/software/batch" query, err := url.ParseQuery(opts.RawQuery()) if err != nil { return nil, err } query.Add("team_name", tmName) - var resp batchSetSoftwareInstallersResponse - if err := c.authenticatedRequestWithQuery(map[string]interface{}{"software": softwareInstallers}, verb, path, &resp, query.Encode()); err != nil { - return nil, err - } - return resp.Packages, nil + return c.applySoftwareInstallers(softwareInstallers, query, opts.DryRun) } func (c *Client) ApplyTeamAppStoreAppsAssociation(tmName string, vppBatchPayload []fleet.VPPBatchPayload, opts fleet.ApplySpecOptions) error { diff --git a/server/service/handler.go b/server/service/handler.go index 21bdd2f7ed57..7012393952bc 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -381,7 +381,10 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.DELETE("/api/_version_/fleet/software/titles/{title_id:[0-9]+}/available_for_install", deleteSoftwareInstallerEndpoint, deleteSoftwareInstallerRequest{}) ue.GET("/api/_version_/fleet/software/install/{install_uuid}/results", getSoftwareInstallResultsEndpoint, getSoftwareInstallResultsRequest{}) + // POST /api/_version_/fleet/software/batch is asynchronous, meaning it will start the process of software download+upload in the background + // and will return a request UUID to be used in GET /api/_version_/fleet/software/batch/{request_uuid} to query for the status of the operation. ue.POST("/api/_version_/fleet/software/batch", batchSetSoftwareInstallersEndpoint, batchSetSoftwareInstallersRequest{}) + ue.GET("/api/_version_/fleet/software/batch/{request_uuid}", batchSetSoftwareInstallersResultEndpoint, batchSetSoftwareInstallersResultRequest{}) // App store software ue.GET("/api/_version_/fleet/software/app_store_apps", getAppStoreAppsEndpoint, getAppStoreAppsRequest{}) diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 9255e95beff0..cb2a97966846 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -10903,6 +10903,10 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { // create an HTTP server to host the software installer handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ruby.deb" { + w.WriteHeader(http.StatusNotFound) + return + } file, err := os.Open(filepath.Join("testdata", "software-installers", "ruby.deb")) require.NoError(t, err) defer file.Close() @@ -10914,11 +10918,28 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { srv := httptest.NewServer(handler) t.Cleanup(srv.Close) + // do a request with a URL that returns a 404. + softwareToInstall = []fleet.SoftwareInstallerPayload{ + {URL: srv.URL + "/not_found.pkg"}, + } + var batchResponse batchSetSoftwareInstallersResponse + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + message := waitBatchSetSoftwareInstallersFailed(t, s, tm.Name, batchResponse.RequestUUID) + require.NotEmpty(t, message) + require.Contains(t, message, fmt.Sprintf("validation failed: software.url Couldn't edit software. URL (\"%s/not_found.pkg\") returned \"Not Found\". Please make sure that URLs are reachable from your Fleet server.", srv.URL)) + // do a request with a valid URL + rubyURL := srv.URL + "/ruby.deb" softwareToInstall = []fleet.SoftwareInstallerPayload{ - {URL: srv.URL}, + {URL: rubyURL}, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages := waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.Equal(t, rubyURL, packages[0].URL) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) // TODO(roberto): test with a variety of response codes @@ -10929,7 +10950,7 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { require.Len(t, titlesResp.SoftwareTitles, 1) // Check that the URL is set to software installers uploaded via batch. require.NotNil(t, titlesResp.SoftwareTitles[0].SoftwarePackage.PackageURL) - require.Equal(t, srv.URL, *titlesResp.SoftwareTitles[0].SoftwarePackage.PackageURL) + require.Equal(t, rubyURL, *titlesResp.SoftwareTitles[0].SoftwarePackage.PackageURL) // check that platform is set when the installer is created mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { @@ -10942,14 +10963,26 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { }) // same payload doesn't modify anything - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.Equal(t, rubyURL, packages[0].URL) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) newTitlesResp := listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &newTitlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) require.Equal(t, titlesResp, newTitlesResp) // setting self-service to true updates the software title metadata softwareToInstall[0].SelfService = true - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.Equal(t, rubyURL, packages[0].URL) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) newTitlesResp = listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &newTitlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) titlesResp.SoftwareTitles[0].SoftwarePackage.SelfService = ptr.Bool(true) @@ -10957,7 +10990,9 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { // empty payload cleans the software items softwareToInstall = []fleet.SoftwareInstallerPayload{} - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Empty(t, packages) titlesResp = listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &titlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) require.Equal(t, 0, titlesResp.Count) @@ -10967,9 +11002,14 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { // Do a request with a valid URL with no team ////////////////////////// softwareToInstall = []fleet.SoftwareInstallerPayload{ - {URL: srv.URL}, + {URL: rubyURL}, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, "", batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.Equal(t, rubyURL, packages[0].URL) + require.Nil(t, packages[0].TeamID) // check the application status on team 0 titlesResp = listSoftwareTitlesResponse{} @@ -10978,14 +11018,24 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { require.Len(t, titlesResp.SoftwareTitles, 1) // same payload doesn't modify anything - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, "", batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.Equal(t, rubyURL, packages[0].URL) + require.Nil(t, packages[0].TeamID) newTitlesResp = listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &newTitlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(0))) require.Equal(t, titlesResp, newTitlesResp) // setting self-service to true updates the software title metadata softwareToInstall[0].SelfService = true - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, "", batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.Equal(t, rubyURL, packages[0].URL) + require.Nil(t, packages[0].TeamID) newTitlesResp = listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &newTitlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(0))) titlesResp.SoftwareTitles[0].SoftwarePackage.SelfService = ptr.Bool(true) @@ -10993,13 +11043,50 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { // empty payload cleans the software items softwareToInstall = []fleet.SoftwareInstallerPayload{} - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, "", batchResponse.RequestUUID) + require.Empty(t, packages) titlesResp = listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &titlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(0))) require.Equal(t, 0, titlesResp.Count) require.Len(t, titlesResp.SoftwareTitles, 0) } +func waitBatchSetSoftwareInstallersCompleted(t *testing.T, s *integrationEnterpriseTestSuite, teamName string, requestUUID string) []fleet.SoftwarePackageResponse { + timeout := time.After(1 * time.Minute) + for { + var batchResultResponse batchSetSoftwareInstallersResultResponse + s.DoJSON("GET", "/api/latest/fleet/software/batch/"+requestUUID, nil, http.StatusOK, &batchResultResponse, "team_name", teamName) + if batchResultResponse.Status == fleet.BatchSetSoftwareInstallersStatusCompleted { + return batchResultResponse.Packages + } + select { + case <-timeout: + t.Fatalf("timeout: %s, %s", teamName, requestUUID) + case <-time.After(500 * time.Millisecond): + // OK, continue + } + } +} + +func waitBatchSetSoftwareInstallersFailed(t *testing.T, s *integrationEnterpriseTestSuite, teamName string, requestUUID string) string { + timeout := time.After(1 * time.Minute) + for { + var batchResultResponse batchSetSoftwareInstallersResultResponse + s.DoJSON("GET", "/api/latest/fleet/software/batch/"+requestUUID, nil, http.StatusOK, &batchResultResponse, "team_name", teamName) + if batchResultResponse.Status == fleet.BatchSetSoftwareInstallersStatusFailed { + require.Empty(t, batchResultResponse.Packages) + return batchResultResponse.Message + } + select { + case <-timeout: + t.Fatalf("timeout: %s, %s", teamName, requestUUID) + case <-time.After(500 * time.Millisecond): + // OK, continue + } + } +} + func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffects() { t := s.T() @@ -11030,7 +11117,14 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffec softwareToInstall := []fleet.SoftwareInstallerPayload{ {URL: srv.URL}, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + var batchResponse batchSetSoftwareInstallersResponse + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages := waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) + require.Equal(t, srv.URL, packages[0].URL) titlesResp := listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &titlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) titleResponse := getSoftwareTitleResponse{} @@ -11068,7 +11162,13 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffec // Switch self-service flag softwareToInstall[0].SelfService = true - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) + require.Equal(t, srv.URL, packages[0].URL) newTitlesResp := listSoftwareTitlesResponse{} s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &newTitlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) require.Equal(t, true, *newTitlesResp.SoftwareTitles[0].SoftwarePackage.SelfService) @@ -11082,7 +11182,13 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffec withUpdatedPreinstallQuery := []fleet.SoftwareInstallerPayload{ {URL: srv.URL, PreInstallQuery: "SELECT * FROM os_version"}, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedPreinstallQuery}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedPreinstallQuery}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) + require.Equal(t, srv.URL, packages[0].URL) titleResponse = getSoftwareTitleResponse{} s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", newTitlesResp.SoftwareTitles[0].ID), nil, http.StatusOK, &titleResponse, "team_id", strconv.Itoa(int(tm.ID))) require.Equal(t, "SELECT * FROM os_version", titleResponse.SoftwareTitle.SoftwarePackage.PreInstallQuery) @@ -11119,7 +11225,13 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffec withUpdatedInstallScript := []fleet.SoftwareInstallerPayload{ {URL: srv.URL, InstallScript: "apt install ruby"}, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedInstallScript}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedInstallScript}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) + require.Equal(t, srv.URL, packages[0].URL) // ensure install count is the same, and uploaded_at hasn't changed s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", newTitlesResp.SoftwareTitles[0].ID), nil, http.StatusOK, &titleResponse, "team_id", strconv.Itoa(int(tm.ID))) @@ -11134,7 +11246,13 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffec trailer = " " // add a character to the response for the installer HTTP call to ensure the file hashes differently // update package - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedInstallScript}, http.StatusOK, "team_name", tm.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedInstallScript}, http.StatusOK, &batchResponse, "team_name", tm.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, tm.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, tm.ID, *packages[0].TeamID) + require.Equal(t, srv.URL, packages[0].URL) // ensure install count is zeroed and uploaded_at HAS changed s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", newTitlesResp.SoftwareTitles[0].ID), nil, http.StatusOK, &titleResponse, "team_id", strconv.Itoa(int(tm.ID))) @@ -11198,7 +11316,15 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersWithPolic URL: srv.URL + "/ruby.deb", }, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", team1.Name) + var batchResponse batchSetSoftwareInstallersResponse + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", team1.Name) + packages := waitBatchSetSoftwareInstallersCompleted(t, s, team1.Name, batchResponse.RequestUUID) + require.Len(t, packages, 1) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, team1.ID, *packages[0].TeamID) + require.Equal(t, srv.URL+"/ruby.deb", packages[0].URL) + // team2 has dummy_installer.pkg and ruby.deb. softwareToInstall = []fleet.SoftwareInstallerPayload{ { @@ -11208,7 +11334,20 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersWithPolic URL: srv.URL + "/ruby.deb", }, } - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", team2.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", team2.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, team2.Name, batchResponse.RequestUUID) + sort.Slice(packages, func(i, j int) bool { + return packages[i].URL < packages[j].URL + }) + require.Len(t, packages, 2) + require.NotNil(t, packages[0].TitleID) + require.NotNil(t, packages[0].TeamID) + require.Equal(t, team2.ID, *packages[0].TeamID) + require.Equal(t, srv.URL+"/dummy_installer.pkg", packages[0].URL) + require.NotNil(t, packages[1].TitleID) + require.NotNil(t, packages[1].TeamID) + require.Equal(t, team2.ID, *packages[1].TeamID) + require.Equal(t, srv.URL+"/ruby.deb", packages[1].URL) // Associate ruby.deb to policy1Team1. resp := listSoftwareTitlesResponse{} @@ -11238,7 +11377,9 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersWithPolic // Get rid of all installers in team1. softwareToInstall = []fleet.SoftwareInstallerPayload{} - s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", team1.Name) + s.DoJSON("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, &batchResponse, "team_name", team1.Name) + packages = waitBatchSetSoftwareInstallersCompleted(t, s, team1.Name, batchResponse.RequestUUID) + require.Len(t, packages, 0) // policy1Team1 should not be associated to any installer. policy1Team1, err = s.ds.Policy(ctx, policy1Team1.ID) diff --git a/server/service/redis_key_value/redis_key_value.go b/server/service/redis_key_value/redis_key_value.go new file mode 100644 index 000000000000..010c24c19cc0 --- /dev/null +++ b/server/service/redis_key_value/redis_key_value.go @@ -0,0 +1,58 @@ +// Package redis_key_value implements a most basic SET & GET key/value store +// where both the key and the value are strings. +package redis_key_value + +import ( + "context" + "errors" + "time" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/datastore/redis" + "github.com/fleetdm/fleet/v4/server/fleet" + redigo "github.com/gomodule/redigo/redis" +) + +// RedisKeyValue is a basic key/value store with SET and GET operations +// Items are removed via expiration (defined in the SET operation). +type RedisKeyValue struct { + pool fleet.RedisPool + testPrefix string // for tests, the key prefix to use to avoid conflicts +} + +// New creates a new RedisKeyValue store. +func New(pool fleet.RedisPool) *RedisKeyValue { + return &RedisKeyValue{pool: pool} +} + +// prefix is used to not collide with other key domains (like live queries or calendar locks). +const prefix = "key_value_" + +// Set creates or overrides the given key with the given value. +// Argument expireTime is used to set the expiration of the item +// (when updating, the expiration of the item is updated). +func (r *RedisKeyValue) Set(ctx context.Context, key string, value string, expireTime time.Duration) error { + conn := redis.ConfigureDoer(r.pool, r.pool.Get()) + defer conn.Close() + + if _, err := redigo.String(conn.Do("SET", r.testPrefix+prefix+key, value, "PX", expireTime.Milliseconds())); err != nil { + return ctxerr.Wrap(ctx, err, "redis failed to set") + } + return nil +} + +// Get returns the value for a given key. +// It returns (nil, nil) if the key doesn't exist. +func (r *RedisKeyValue) Get(ctx context.Context, key string) (*string, error) { + conn := redis.ConfigureDoer(r.pool, r.pool.Get()) + defer conn.Close() + + res, err := redigo.String(conn.Do("GET", r.testPrefix+prefix+key)) + if errors.Is(err, redigo.ErrNil) { + return nil, nil + } + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "redis failed to get") + } + return &res, nil +} diff --git a/server/service/redis_key_value/redis_key_value_test.go b/server/service/redis_key_value/redis_key_value_test.go new file mode 100644 index 000000000000..5f410e4a4918 --- /dev/null +++ b/server/service/redis_key_value/redis_key_value_test.go @@ -0,0 +1,92 @@ +package redis_key_value + +import ( + "context" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/datastore/redis/redistest" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/test" + "github.com/stretchr/testify/require" +) + +func TestRedisKeyValue(t *testing.T) { + for _, f := range []func(*testing.T, *RedisKeyValue){ + testSetGet, + } { + t.Run(test.FunctionName(f), func(t *testing.T) { + t.Run("standalone", func(t *testing.T) { + kv := setupRedis(t, false, false) + f(t, kv) + }) + t.Run("cluster", func(t *testing.T) { + kv := setupRedis(t, true, true) + f(t, kv) + }) + }) + } +} + +func setupRedis(t testing.TB, cluster, redir bool) *RedisKeyValue { + pool := redistest.SetupRedis(t, t.Name(), cluster, redir, true) + return newRedisKeyValueForTest(t, pool) +} + +type testName interface { + Name() string +} + +func newRedisKeyValueForTest(t testName, pool fleet.RedisPool) *RedisKeyValue { + return &RedisKeyValue{ + pool: pool, + testPrefix: t.Name() + ":", + } +} + +func testSetGet(t *testing.T, kv *RedisKeyValue) { + ctx := context.Background() + + result, err := kv.Get(ctx, "foo") + require.NoError(t, err) + require.Nil(t, result) + + err = kv.Set(ctx, "foo", "bar", 5*time.Second) + require.NoError(t, err) + + result, err = kv.Get(ctx, "foo") + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "bar", *result) + + err = kv.Set(ctx, "foo", "zoo", 5*time.Second) + require.NoError(t, err) + + result, err = kv.Get(ctx, "foo") + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "zoo", *result) + + err = kv.Set(ctx, "boo", "bar", 2*time.Second) + require.NoError(t, err) + result, err = kv.Get(ctx, "boo") + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "bar", *result) + + time.Sleep(3 * time.Second) + result, err = kv.Get(ctx, "boo") + require.NoError(t, err) + require.Nil(t, result) + + // Updating an item, updates the expiration time. + err = kv.Set(ctx, "test", "foo", 2*time.Second) + require.NoError(t, err) + err = kv.Set(ctx, "test", "foo", 10*time.Second) + require.NoError(t, err) + time.Sleep(5 * time.Second) + result, err = kv.Get(ctx, "test") + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "foo", *result) +} diff --git a/server/service/software_installers.go b/server/service/software_installers.go index 0542d769c805..b10b6a6f4c7f 100644 --- a/server/service/software_installers.go +++ b/server/service/software_installers.go @@ -546,27 +546,64 @@ type batchSetSoftwareInstallersRequest struct { } type batchSetSoftwareInstallersResponse struct { - Packages []fleet.SoftwarePackageResponse `json:"packages"` - Err error `json:"error,omitempty"` + RequestUUID string `json:"request_uuid"` + Err error `json:"error,omitempty"` } func (r batchSetSoftwareInstallersResponse) error() error { return r.Err } func batchSetSoftwareInstallersEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { req := request.(*batchSetSoftwareInstallersRequest) - packages, err := svc.BatchSetSoftwareInstallers(ctx, req.TeamName, req.Software, req.DryRun) + requestUUID, err := svc.BatchSetSoftwareInstallers(ctx, req.TeamName, req.Software, req.DryRun) if err != nil { return batchSetSoftwareInstallersResponse{Err: err}, nil } - return batchSetSoftwareInstallersResponse{Packages: packages}, nil + return batchSetSoftwareInstallersResponse{RequestUUID: requestUUID}, nil } -func (svc *Service) BatchSetSoftwareInstallers(ctx context.Context, tmName string, payloads []fleet.SoftwareInstallerPayload, dryRun bool) ([]fleet.SoftwarePackageResponse, error) { +func (svc *Service) BatchSetSoftwareInstallers(ctx context.Context, tmName string, payloads []fleet.SoftwareInstallerPayload, dryRun bool) (string, error) { // skipauth: No authorization check needed due to implementation returning // only license error. svc.authz.SkipAuthorization(ctx) - return nil, fleet.ErrMissingLicense + return "", fleet.ErrMissingLicense +} + +type batchSetSoftwareInstallersResultRequest struct { + RequestUUID string `url:"request_uuid"` + TeamName string `query:"team_name,optional"` + DryRun bool `query:"dry_run,optional"` // if true, apply validation but do not save changes +} + +type batchSetSoftwareInstallersResultResponse struct { + Status string `json:"status"` + Message string `json:"message"` + Packages []fleet.SoftwarePackageResponse `json:"packages"` + + Err error `json:"error,omitempty"` +} + +func (r batchSetSoftwareInstallersResultResponse) error() error { return r.Err } + +func batchSetSoftwareInstallersResultEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*batchSetSoftwareInstallersResultRequest) + status, message, packages, err := svc.GetBatchSetSoftwareInstallersResult(ctx, req.TeamName, req.RequestUUID, req.DryRun) + if err != nil { + return batchSetSoftwareInstallersResultResponse{Err: err}, nil + } + return batchSetSoftwareInstallersResultResponse{ + Status: status, + Message: message, + Packages: packages, + }, nil +} + +func (svc *Service) GetBatchSetSoftwareInstallersResult(ctx context.Context, tmName string, requestUUID string, dryRun bool) (string, string, []fleet.SoftwarePackageResponse, error) { + // skipauth: No authorization check needed due to implementation returning + // only license error. + svc.authz.SkipAuthorization(ctx) + + return "", "", nil, fleet.ErrMissingLicense } ////////////////////////////////////////////////////////////////////////////// diff --git a/server/service/testing_utils.go b/server/service/testing_utils.go index 29edb79ae326..8012ae3ffbf5 100644 --- a/server/service/testing_utils.go +++ b/server/service/testing_utils.go @@ -34,6 +34,7 @@ import ( "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/service/async" "github.com/fleetdm/fleet/v4/server/service/mock" + "github.com/fleetdm/fleet/v4/server/service/redis_key_value" "github.com/fleetdm/fleet/v4/server/service/redis_lock" "github.com/fleetdm/fleet/v4/server/sso" "github.com/fleetdm/fleet/v4/server/test" @@ -72,6 +73,7 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf softwareInstallStore fleet.SoftwareInstallerStore bootstrapPackageStore fleet.MDMBootstrapPackageStore distributedLock fleet.Lock + keyValueStore fleet.KeyValueStore ) if len(opts) > 0 { if opts[0].Clock != nil { @@ -79,6 +81,10 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf } } + if len(opts) > 0 && opts[0].KeyValueStore != nil { + keyValueStore = opts[0].KeyValueStore + } + task := async.NewTask(ds, nil, c, config.OsqueryConfig{}) if len(opts) > 0 { if opts[0].Task != nil { @@ -99,6 +105,7 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf ssoStore = sso.NewSessionStore(opts[0].Pool) profMatcher = apple_mdm.NewProfileMatcher(opts[0].Pool) distributedLock = redis_lock.NewLock(opts[0].Pool) + keyValueStore = redis_key_value.New(opts[0].Pool) } if opts[0].ProfileMatcher != nil { profMatcher = opts[0].ProfileMatcher @@ -203,6 +210,7 @@ func newTestServiceWithConfig(t *testing.T, ds fleet.Datastore, fleetConfig conf softwareInstallStore, bootstrapPackageStore, distributedLock, + keyValueStore, ) if err != nil { panic(err) @@ -317,6 +325,7 @@ type TestServerOpts struct { NoCacheDatastore bool SoftwareInstallStore fleet.SoftwareInstallerStore BootstrapPackageStore fleet.MDMBootstrapPackageStore + KeyValueStore fleet.KeyValueStore } func RunServerForTestsWithDS(t *testing.T, ds fleet.Datastore, opts ...*TestServerOpts) (map[string]fleet.User, *httptest.Server) {