From 632ba549dedbaf1e3a7506981454667dd0990e37 Mon Sep 17 00:00:00 2001 From: Ian Littman Date: Tue, 17 Sep 2024 11:00:46 -0500 Subject: [PATCH] Add installer edit side effects to batch installer update (via GitOps) (#22100) #21612 # Checklist for submitter - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files) for more information. - [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements) - [x] Added/updated tests --------- Co-authored-by: RachelElysia Co-authored-by: Luke Heath Co-authored-by: Jacob Shandling Co-authored-by: Victor Lyuboslavsky --- changes/21612-edit-software-gitops | 1 + server/datastore/mysql/software_installers.go | 101 +++++++++--- server/service/integration_enterprise_test.go | 148 ++++++++++++++++++ 3 files changed, 230 insertions(+), 20 deletions(-) create mode 100644 changes/21612-edit-software-gitops diff --git a/changes/21612-edit-software-gitops b/changes/21612-edit-software-gitops new file mode 100644 index 000000000000..9a157286d49b --- /dev/null +++ b/changes/21612-edit-software-gitops @@ -0,0 +1 @@ +* Reset install counts and cancel pending installs/uninstalls when GitOps installer updates change package contents diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index a4fd79270bd8..0f20094a6971 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -467,32 +467,36 @@ func (ds *Datastore) InsertSoftwareInstallRequest(ctx context.Context, hostID ui func (ds *Datastore) ProcessInstallerUpdateSideEffects(ctx context.Context, installerID uint, wasMetadataUpdated bool, wasPackageUpdated bool) error { return ds.withTx(ctx, func(tx sqlx.ExtContext) error { - if wasMetadataUpdated || wasPackageUpdated { // cancel pending installs/uninstalls - // TODO make this less naive; this assumes that installs/uninstalls execute and report back immediately - _, err := tx.ExecContext(ctx, `DELETE FROM host_script_results WHERE execution_id IN ( - SELECT execution_id FROM host_software_installs WHERE software_installer_id = ? AND status = "pending_uninstall" + return ds.runInstallerUpdateSideEffectsInTransaction(ctx, tx, installerID, wasMetadataUpdated, wasPackageUpdated) + }) +} + +func (ds *Datastore) runInstallerUpdateSideEffectsInTransaction(ctx context.Context, tx sqlx.ExtContext, installerID uint, wasMetadataUpdated bool, wasPackageUpdated bool) error { + if wasMetadataUpdated || wasPackageUpdated { // cancel pending installs/uninstalls + // TODO make this less naive; this assumes that installs/uninstalls execute and report back immediately + _, err := tx.ExecContext(ctx, `DELETE FROM host_script_results WHERE execution_id IN ( + SELECT execution_id FROM host_software_installs WHERE software_installer_id = ? AND status = 'pending_uninstall' )`, installerID) - if err != nil { - return ctxerr.Wrap(ctx, err, "delete pending uninstall scripts") - } + if err != nil { + return ctxerr.Wrap(ctx, err, "delete pending uninstall scripts") + } - _, err = tx.ExecContext(ctx, `DELETE FROM host_software_installs - WHERE software_installer_id = ? AND status IN("pending_install", "pending_uninstall")`, installerID) - if err != nil { - return ctxerr.Wrap(ctx, err, "delete pending host software installs/uninstalls") - } + _, err = tx.ExecContext(ctx, `DELETE FROM host_software_installs + WHERE software_installer_id = ? AND status IN('pending_install', 'pending_uninstall')`, installerID) + if err != nil { + return ctxerr.Wrap(ctx, err, "delete pending host software installs/uninstalls") } + } - if wasPackageUpdated { // hide existing install counts - _, err := tx.ExecContext(ctx, `UPDATE host_software_installs SET removed = TRUE + if wasPackageUpdated { // hide existing install counts + _, err := tx.ExecContext(ctx, `UPDATE host_software_installs SET removed = TRUE WHERE software_installer_id = ? AND status IS NOT NULL AND host_deleted_at IS NULL`, installerID) - if err != nil { - return ctxerr.Wrap(ctx, err, "hide existing install counts") - } + if err != nil { + return ctxerr.Wrap(ctx, err, "hide existing install counts") } + } - return nil - }) + return nil } func (ds *Datastore) InsertSoftwareUninstallRequest(ctx context.Context, executionID string, hostID uint, softwareInstallerID uint) error { @@ -817,6 +821,17 @@ WHERE title_id NOT IN (?) ` + const checkExistingInstaller = ` +SELECT id, +storage_id != ? is_package_modified, +install_script_content_id != ? OR uninstall_script_content_id != ? OR pre_install_query != ? OR +COALESCE(post_install_script_content_id != ? OR + (post_install_script_content_id IS NULL AND ? IS NOT NULL) OR + (? IS NULL AND post_install_script_content_id IS NOT NULL) +, FALSE) is_metadata_modified FROM software_installers +WHERE global_or_team_id = ? AND title_id IN (SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = '') +` + const insertNewOrEditedInstaller = ` INSERT INTO software_installers ( team_id, @@ -947,6 +962,36 @@ WHERE global_or_team_id = ? postInstallScriptID = &insertID } + wasUpdatedArgs := []interface{}{ + // package update + installer.StorageID, + // metadata update + installScriptID, + uninstallScriptID, + installer.PreInstallQuery, + postInstallScriptID, + postInstallScriptID, + postInstallScriptID, + // WHERE clause + globalOrTeamID, + installer.Title, + installer.Source, + } + + // pull existing installer state if it exists so we can diff for side effects post-update + type existingInstallerUpdateCheckResult struct { + InstallerID uint `db:"id"` + IsPackageModified bool `db:"is_package_modified"` + IsMetadataModified bool `db:"is_metadata_modified"` + } + var existing []existingInstallerUpdateCheckResult + err = sqlx.SelectContext(ctx, tx, &existing, checkExistingInstaller, wasUpdatedArgs...) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return ctxerr.Wrapf(ctx, err, "checking for existing installer with name %q", installer.Filename) + } + } + args := []interface{}{ tmID, globalOrTeamID, @@ -968,11 +1013,27 @@ WHERE global_or_team_id = ? installer.URL, strings.Join(installer.PackageIDs, ","), } + upsertQuery := insertNewOrEditedInstaller + if len(existing) > 0 && existing[0].IsPackageModified { // update uploaded_at for updated installer package + upsertQuery = fmt.Sprintf("%s, uploaded_at = NOW()", upsertQuery) + } - if _, err := tx.ExecContext(ctx, insertNewOrEditedInstaller, args...); err != nil { + if _, err := tx.ExecContext(ctx, upsertQuery, args...); err != nil { return ctxerr.Wrapf(ctx, err, "insert new/edited installer with name %q", installer.Filename) } + // perform side effects if this was an update + if len(existing) > 0 { + if err := ds.runInstallerUpdateSideEffectsInTransaction( + ctx, + tx, + existing[0].InstallerID, + existing[0].IsMetadataModified, + existing[0].IsPackageModified, + ); err != nil { + return ctxerr.Wrapf(ctx, err, "processing installer with name %q", installer.Filename) + } + } } if err := sqlx.SelectContext(ctx, tx, &insertedSoftwareInstallers, loadInsertedSoftwareInstallers, globalOrTeamID); err != nil { diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index e5d9316ae44d..9255e95beff0 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -11000,6 +11000,154 @@ func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallers() { require.Len(t, titlesResp.SoftwareTitles, 0) } +func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersSideEffects() { + t := s.T() + + // create a team + tm, err := s.ds.NewTeam(context.Background(), &fleet.Team{ + Name: t.Name(), + Description: "desc", + }) + require.NoError(t, err) + + // create an HTTP server to host the software installer + trailer := "" + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + file, err := os.Open(filepath.Join("testdata", "software-installers", "ruby.deb")) + require.NoError(t, err) + defer file.Close() + w.Header().Set("Content-Type", "application/vnd.debian.binary-package") + _, err = io.Copy(w, file) + require.NoError(t, err) + _, err = w.Write([]byte(trailer)) + require.NoError(t, err) + }) + + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + // set up software to install + softwareToInstall := []fleet.SoftwareInstallerPayload{ + {URL: srv.URL}, + } + s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + titlesResp := listSoftwareTitlesResponse{} + s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &titlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) + titleResponse := getSoftwareTitleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", titlesResp.SoftwareTitles[0].ID), nil, http.StatusOK, &titleResponse, "team_id", strconv.Itoa(int(tm.ID))) + uploadedAt := titleResponse.SoftwareTitle.SoftwarePackage.UploadedAt + + // create a host that doesn't have fleetd installed + h, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now().Add(-1 * time.Minute), + OsqueryHostID: ptr.String(t.Name() + uuid.New().String()), + NodeKey: ptr.String(t.Name() + uuid.New().String()), + Hostname: fmt.Sprintf("%sfoo.local", t.Name()), + Platform: "linux", + }) + require.NoError(t, err) + err = s.ds.AddHostsToTeam(context.Background(), &tm.ID, []uint{h.ID}) + require.NoError(t, err) + h.TeamID = &tm.ID + + // host installs fleetd + orbitKey := setOrbitEnrollment(t, h, s.ds) + h.OrbitNodeKey = &orbitKey + + // install software + installResp := installSoftwareResponse{} + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/%d/install", h.ID, titlesResp.SoftwareTitles[0].ID), nil, http.StatusAccepted, &installResp) + + // Get the install response, should be pending + getHostSoftwareResp := getHostSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &getHostSoftwareResp) + require.Equal(t, fleet.SoftwareInstallPending, *getHostSoftwareResp.Software[0].Status) + + // Switch self-service flag + softwareToInstall[0].SelfService = true + s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: softwareToInstall}, http.StatusOK, "team_name", tm.Name) + newTitlesResp := listSoftwareTitlesResponse{} + s.DoJSON("GET", "/api/v1/fleet/software/titles", nil, http.StatusOK, &newTitlesResp, "available_for_install", "true", "team_id", strconv.Itoa(int(tm.ID))) + require.Equal(t, true, *newTitlesResp.SoftwareTitles[0].SoftwarePackage.SelfService) + + // Install should still be pending + afterSelfServiceHostResp := getHostSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &afterSelfServiceHostResp) + require.Equal(t, fleet.SoftwareInstallPending, *getHostSoftwareResp.Software[0].Status) + + // update pre-install query + withUpdatedPreinstallQuery := []fleet.SoftwareInstallerPayload{ + {URL: srv.URL, PreInstallQuery: "SELECT * FROM os_version"}, + } + s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedPreinstallQuery}, http.StatusOK, "team_name", tm.Name) + titleResponse = getSoftwareTitleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", newTitlesResp.SoftwareTitles[0].ID), nil, http.StatusOK, &titleResponse, "team_id", strconv.Itoa(int(tm.ID))) + require.Equal(t, "SELECT * FROM os_version", titleResponse.SoftwareTitle.SoftwarePackage.PreInstallQuery) + require.Equal(t, uint(0), titleResponse.SoftwareTitle.SoftwarePackage.Status.PendingInstall) + + // install should no longer be pending + afterPreinstallHostResp := getHostSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &afterPreinstallHostResp) + require.Nil(t, afterPreinstallHostResp.Software[0].Status) + + // install software fully + s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/%d/install", h.ID, titlesResp.SoftwareTitles[0].ID), nil, http.StatusAccepted, &installResp) + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &getHostSoftwareResp) + installUUID := getHostSoftwareResp.Software[0].SoftwarePackage.LastInstall.InstallUUID + s.Do("POST", "/api/fleet/orbit/software_install/result", json.RawMessage(fmt.Sprintf(`{ + "orbit_node_key": %q, + "install_uuid": %q, + "pre_install_condition_output": "ok", + "install_script_exit_code": 0, + "install_script_output": "ok" + }`, *h.OrbitNodeKey, installUUID)), http.StatusNoContent) + + // ensure install count is updated + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", newTitlesResp.SoftwareTitles[0].ID), nil, http.StatusOK, &titleResponse, "team_id", strconv.Itoa(int(tm.ID))) + require.Equal(t, uint(1), titleResponse.SoftwareTitle.SoftwarePackage.Status.Installed) + require.Equal(t, uint(0), titleResponse.SoftwareTitle.SoftwarePackage.Status.PendingInstall) + + // install should show as complete + hostResp := getHostSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &hostResp) + require.Equal(t, fleet.SoftwareInstalled, *hostResp.Software[0].Status) + + // update install script + withUpdatedInstallScript := []fleet.SoftwareInstallerPayload{ + {URL: srv.URL, InstallScript: "apt install ruby"}, + } + s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedInstallScript}, http.StatusOK, "team_name", tm.Name) + + // ensure install count is the same, and uploaded_at hasn't changed + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", newTitlesResp.SoftwareTitles[0].ID), nil, http.StatusOK, &titleResponse, "team_id", strconv.Itoa(int(tm.ID))) + require.Equal(t, uint(1), titleResponse.SoftwareTitle.SoftwarePackage.Status.Installed) + require.Equal(t, uint(0), titleResponse.SoftwareTitle.SoftwarePackage.Status.PendingInstall) + require.Equal(t, uploadedAt, titleResponse.SoftwareTitle.SoftwarePackage.UploadedAt) + + // install should still show as complete + hostResp = getHostSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &hostResp) + require.Equal(t, fleet.SoftwareInstalled, *hostResp.Software[0].Status) + + trailer = " " // add a character to the response for the installer HTTP call to ensure the file hashes differently + // update package + s.Do("POST", "/api/latest/fleet/software/batch", batchSetSoftwareInstallersRequest{Software: withUpdatedInstallScript}, http.StatusOK, "team_name", tm.Name) + + // ensure install count is zeroed and uploaded_at HAS changed + s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/software/titles/%d", newTitlesResp.SoftwareTitles[0].ID), nil, http.StatusOK, &titleResponse, "team_id", strconv.Itoa(int(tm.ID))) + require.Equal(t, uint(0), titleResponse.SoftwareTitle.SoftwarePackage.Status.Installed) + require.Equal(t, uint(0), titleResponse.SoftwareTitle.SoftwarePackage.Status.PendingInstall) + require.NotEqual(t, uploadedAt, titleResponse.SoftwareTitle.SoftwarePackage.UploadedAt) + + // install should be nulled out + hostResp = getHostSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", h.ID), nil, http.StatusOK, &hostResp) + require.Nil(t, hostResp.Software[0].Status) +} + func (s *integrationEnterpriseTestSuite) TestBatchSetSoftwareInstallersWithPoliciesAssociated() { ctx := context.Background() t := s.T()