[Non-Root] Add support for running without root permissions on Linux (#…

…3598)

* Work on non-root.

* Only create group and user if not existing.

* More work on non-root for darwin.

* Fix FindUID.

* More fixes.

* Fix service. Add linux user commands.

* Fix control socket path.

* Trim unix://.

* Fix permissions on socket path create.

* More socket fixes.

* Use MkdirAll.

* Fix SysProcAttr.

* Only Linux.

* mage check

* Add non-root install integration tests.

* Fix imports.

* Fix windows build.

* Testing fixes.

* Fix non-root base path test.

* Empty commit.

* More non-root fixes.

* Fix issue from merge.

* Upgrade unpack prevent world permissions.

* Add group permissions for logs in non-root mode.

* Fix permissions for internal logger.

* Fixes from code review.

* Change to using a structure.

* More code review fixes.

* Update to fmt.Errorf.

internal/pkg/agent/application/info/state.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
+	"github.com/elastic/elastic-agent/pkg/utils"
 )
 
 // MarkerFileName is the name of the file that's created by
@@ -28,11 +29,10 @@ func RunningInstalled() bool {
 	return true
 }
 
-func CreateInstallMarker(topPath string) error {
+func CreateInstallMarker(topPath string, ownership utils.FileOwner) error {
 	markerFilePath := filepath.Join(topPath, MarkerFileName)
 	if _, err := os.Create(markerFilePath); err != nil {
 		return err
 	}
-
-	return nil
+	return fixInstallMarkerPermissions(markerFilePath, ownership)
 }

internal/pkg/agent/application/info/state_unix.go

@@ -0,0 +1,22 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+//go:build !windows
+
+package info
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/elastic/elastic-agent/pkg/utils"
+)
+
+func fixInstallMarkerPermissions(markerFilePath string, ownership utils.FileOwner) error {
+	err := os.Chown(markerFilePath, ownership.UID, ownership.GID)
+	if err != nil {
+		return fmt.Errorf("failed to chown %d:%d %s: %w", ownership.UID, ownership.GID, markerFilePath, err)
+	}
+	return nil
+}

internal/pkg/agent/application/info/state_windows.go

@@ -0,0 +1,16 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+//go:build windows
+
+package info
+
+import (
+	"github.com/elastic/elastic-agent/pkg/utils"
+)
+
+func fixInstallMarkerPermissions(markerFilePath string, ownership utils.FileOwner) error {
+	// TODO(blakerouse): Fix the market permissions on Windows.
+	return nil
+}

internal/pkg/agent/application/paths/paths.go

@@ -17,6 +17,12 @@ const (
 	// ControlSocketPath is the control socket path used when installed.
 	ControlSocketPath = "unix:///run/elastic-agent.sock"
 
+	// ControlSocketUnprivilegedPath is the control socket path used when installed as non-root.
+	// This must exist inside of a directory in '/run/' because the permissions need to be set
+	// on that directory during installation time, because once the service is spawned it will not
+	// have permissions to create the socket in the '/run/' directory.
+	ControlSocketUnprivilegedPath = "unix:///run/elastic-agent/elastic-agent.sock"
+
 	// ShipperSocketPipePattern is the socket path used when installed for a shipper pipe.
 	ShipperSocketPipePattern = "unix:///run/elastic-agent-%s-pipe.sock"
 

internal/pkg/agent/application/paths/paths_darwin.go

@@ -17,6 +17,12 @@ const (
 	// ControlSocketPath is the control socket path used when installed.
 	ControlSocketPath = "unix:///var/run/elastic-agent.sock"
 
+	// ControlSocketUnprivilegedPath is the control socket path used when installed as non-root.
+	// This must exist inside of a directory in '/var/run/' because the permissions need to be set
+	// on that directory during installation time, because once the service is spawned it will not
+	// have permissions to create the socket in the '/var/run/' directory.
+	ControlSocketUnprivilegedPath = "unix:///var/run/elastic-agent/elastic-agent.sock"
+
 	// ShipperSocketPipePattern is the socket path used when installed for a shipper pipe.
 	ShipperSocketPipePattern = "unix:///var/run/elastic-agent-%s-pipe.sock"
 

internal/pkg/agent/application/paths/paths_windows.go

@@ -22,6 +22,9 @@ const (
 	// ControlSocketPath is the control socket path used when installed.
 	ControlSocketPath = `\\.\pipe\elastic-agent-system`
 
+	// ControlSocketUnprivilegedPath is the control socket path used when installed as non-root.
+	ControlSocketUnprivilegedPath = ControlSocketPath
+
 	// ShipperSocketPipePattern is the socket path used when installed for a shipper pipe.
 	ShipperSocketPipePattern = `\\.\pipe\elastic-agent-%s-pipe.sock`
 

internal/pkg/agent/application/upgrade/rollback.go

@@ -12,16 +12,17 @@ import (
 	"strings"
 	"time"
 
-	"github.com/elastic/elastic-agent/pkg/control"
-	"github.com/elastic/elastic-agent/pkg/control/v2/client"
-
 	"github.com/hashicorp/go-multierror"
+	"google.golang.org/grpc"
 
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/errors"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/install"
 	"github.com/elastic/elastic-agent/internal/pkg/core/backoff"
+	"github.com/elastic/elastic-agent/pkg/control"
+	"github.com/elastic/elastic-agent/pkg/control/v2/client"
 	"github.com/elastic/elastic-agent/pkg/core/logger"
+	"github.com/elastic/elastic-agent/pkg/utils"
 )
 
 const (
@@ -137,7 +138,9 @@ func InvokeWatcher(log *logger.Logger) error {
 func restartAgent(ctx context.Context, log *logger.Logger) error {
 	restartViaDaemonFn := func(ctx context.Context) error {
 		c := client.New()
-		err := c.Connect(ctx)
+		connectCtx, connectCancel := context.WithTimeout(ctx, 3*time.Second)
+		defer connectCancel()
+		err := c.Connect(connectCtx, grpc.WithBlock(), grpc.WithDisableRetry())
 		if err != nil {
 			return errors.New(err, "failed communicating to running daemon", errors.TypeNetwork, errors.M("socket", control.Address()))
 		}
@@ -163,6 +166,7 @@ func restartAgent(ctx context.Context, log *logger.Logger) error {
 
 	signal := make(chan struct{})
 	backExp := backoff.NewExpBackoff(signal, restartBackoffInit, restartBackoffMax)
+	root, _ := utils.HasRoot() // error ignored
 
 	for restartAttempt := 1; restartAttempt <= maxRestartCount; restartAttempt++ {
 		backExp.Wait()
@@ -175,19 +179,21 @@ func restartAgent(ctx context.Context, log *logger.Logger) error {
 		}
 		log.Warnf("Failed to restart agent via control protocol: %s", err.Error())
 
-		// Next, try to restart Agent via the service.
-		log.Infof("Restarting Agent via service; attempt %d of %d", restartAttempt, maxRestartCount)
-		err = restartViaServiceFn(ctx)
-		if err == nil {
-			break
+		// Next, try to restart Agent via the service. (only if root)
+		if root {
+			log.Infof("Restarting Agent via service; attempt %d of %d", restartAttempt, maxRestartCount)
+			err = restartViaServiceFn(ctx)
+			if err == nil {
+				break
+			}
+			log.Warnf("Failed to restart agent via service: %s", err.Error())
 		}
 
 		if restartAttempt == maxRestartCount {
 			log.Error("Failed to restart agent after final attempt")
 			return err
 		}
-
-		log.Warnf("Failed to restart agent via service: %s; will try again in %v", err.Error(), backExp.NextWait())
+		log.Warnf("Failed to restart agent; will try again in %v", backExp.NextWait())
 	}
 
 	close(signal)

internal/pkg/agent/application/upgrade/step_unpack.go

@@ -85,11 +85,13 @@ func unzip(log *logger.Logger, archivePath string) (string, error) {
 
 		if f.FileInfo().IsDir() {
 			log.Debugw("Unpacking directory", "archive", "zip", "file.path", path)
-			_ = os.MkdirAll(path, f.Mode())
+			// remove any world permissions from the directory
+			_ = os.MkdirAll(path, f.Mode()&0770)
 		} else {
 			log.Debugw("Unpacking file", "archive", "zip", "file.path", path)
-			_ = os.MkdirAll(filepath.Dir(path), f.Mode())
-			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+			// remove any world permissions from the directory/file
+			_ = os.MkdirAll(filepath.Dir(path), f.Mode()&0770)
+			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()&0770)
 			if err != nil {
 				return err
 			}
@@ -190,11 +192,13 @@ func untar(log *logger.Logger, version string, archivePath string) (string, erro
 		case mode.IsRegular():
 			log.Debugw("Unpacking file", "archive", "tar", "file.path", abs)
 			// just to be sure, it should already be created by Dir type
-			if err := os.MkdirAll(filepath.Dir(abs), 0755); err != nil {
+			// remove any world permissions from the directory
+			if err := os.MkdirAll(filepath.Dir(abs), mode.Perm()&0770); err != nil {
 				return "", errors.New(err, "TarInstaller: creating directory for file "+abs, errors.TypeFilesystem, errors.M(errors.MetaKeyPath, abs))
 			}
 
-			wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm())
+			// remove any world permissions from the file
+			wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm()&0770)
 			if err != nil {
 				return "", errors.New(err, "TarInstaller: creating file "+abs, errors.TypeFilesystem, errors.M(errors.MetaKeyPath, abs))
 			}
@@ -209,7 +213,8 @@ func untar(log *logger.Logger, version string, archivePath string) (string, erro
 			}
 		case mode.IsDir():
 			log.Debugw("Unpacking directory", "archive", "tar", "file.path", abs)
-			if err := os.MkdirAll(abs, 0755); err != nil {
+			// remove any world permissions from the directory
+			if err := os.MkdirAll(abs, mode.Perm()&0770); err != nil {
 				return "", errors.New(err, "TarInstaller: creating directory for file "+abs, errors.TypeFilesystem, errors.M(errors.MetaKeyPath, abs))
 			}
 		default:

internal/pkg/agent/cmd/enroll_cmd.go

@@ -15,6 +15,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/elastic/elastic-agent/pkg/utils"
+
 	"github.com/elastic/elastic-agent/pkg/control/v2/client"
 
 	"go.elastic.co/apm"
@@ -260,7 +262,7 @@ func (c *enrollCmd) Execute(ctx context.Context, streams *cli.IOStreams) error {
 	}
 
 	if c.options.FixPermissions {
-		err = install.FixPermissions(paths.Top())
+		err = install.FixPermissions(paths.Top(), utils.CurrentFileOwner())
 		if err != nil {
 			return errors.New(err, "failed to fix permissions")
 		}

internal/pkg/agent/cmd/install.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 
 	"github.com/spf13/cobra"
 
@@ -21,7 +22,10 @@ import (
 	"github.com/elastic/elastic-agent/pkg/utils"
 )
 
-const flagInstallBasePath = "base-path"
+const (
+	flagInstallBasePath     = "base-path"
+	flagInstallUnprivileged = "unprivileged"
+)
 
 func newInstallCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command {
 	cmd := &cobra.Command{
@@ -43,6 +47,7 @@ would like the Agent to operate.
 	cmd.Flags().BoolP("force", "f", false, "Force overwrite the current installation and do not prompt for confirmation")
 	cmd.Flags().BoolP("non-interactive", "n", false, "Install Elastic Agent in non-interactive mode which will not prompt on missing parameters but fails instead.")
 	cmd.Flags().String(flagInstallBasePath, paths.DefaultBasePath, "The path where the Elastic Agent will be installed. It must be an absolute path.")
+	cmd.Flags().Bool(flagInstallUnprivileged, false, "Installed Elastic Agent will create an 'elastic-agent' user and run as that user.")
 	addEnrollFlags(cmd)
 
 	return cmd
@@ -67,6 +72,12 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 		return fmt.Errorf("unable to perform install command, not executed with %s permissions", utils.PermissionUser)
 	}
 
+	// only support Linux at the moment
+	unprivileged, _ := cmd.Flags().GetBool(flagInstallUnprivileged)
+	if unprivileged && runtime.GOOS != "linux" {
+		return fmt.Errorf("unable to perform install command, unprivileged is currently only supported on Linux")
+	}
+
 	topPath := paths.InstallPath(basePath)
 
 	status, reason := install.Status(topPath)
@@ -175,9 +186,10 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 
 	progBar := install.CreateAndStartNewSpinner(streams.Out, "Installing Elastic Agent...")
 
+	var ownership utils.FileOwner
 	cfgFile := paths.ConfigFile()
 	if status != install.PackageInstall {
-		err = install.Install(cfgFile, topPath, progBar, streams)
+		ownership, err = install.Install(cfgFile, topPath, unprivileged, progBar, streams)
 		if err != nil {
 			return fmt.Errorf("error installing package: %w", err)
 		}
@@ -225,6 +237,10 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 		enrollCmd.Stdin = os.Stdin
 		enrollCmd.Stdout = os.Stdout
 		enrollCmd.Stderr = os.Stderr
+		err = enrollCmdExtras(enrollCmd, ownership)
+		if err != nil {
+			return err
+		}
 
 		progBar.Describe("Enrolling Elastic Agent with Fleet")
 		err = enrollCmd.Start()
@@ -243,7 +259,7 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 		progBar.Describe("Enroll Completed")
 	}
 
-	if err := info.CreateInstallMarker(topPath); err != nil {
+	if err := info.CreateInstallMarker(topPath, ownership); err != nil {
 		return fmt.Errorf("failed to create install marker: %w", err)
 	}
 

internal/pkg/agent/cmd/install_enroll.go

@@ -0,0 +1,24 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+//go:build !windows
+
+package cmd
+
+import (
+	"os/exec"
+	"syscall"
+
+	"github.com/elastic/elastic-agent/pkg/utils"
+)
+
+func enrollCmdExtras(cmd *exec.Cmd, ownership utils.FileOwner) error {
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Credential: &syscall.Credential{
+			Uid: uint32(ownership.UID),
+			Gid: uint32(ownership.GID),
+		},
+	}
+	return nil
+}

internal/pkg/agent/cmd/install_enroll_windows.go

@@ -0,0 +1,18 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+//go:build windows
+
+package cmd
+
+import (
+	"os/exec"
+
+	"github.com/elastic/elastic-agent/pkg/utils"
+)
+
+func enrollCmdExtras(cmd *exec.Cmd, ownership utils.FileOwner) error {
+	// TODO: Add ability to call enroll as non-Administrator on Windows.
+	return nil
+}

internal/pkg/agent/cmd/run.go

@@ -49,6 +49,7 @@ import (
 	"github.com/elastic/elastic-agent/pkg/component"
 	"github.com/elastic/elastic-agent/pkg/control/v2/server"
 	"github.com/elastic/elastic-agent/pkg/core/logger"
+	"github.com/elastic/elastic-agent/pkg/utils"
 	"github.com/elastic/elastic-agent/version"
 )
 
@@ -612,7 +613,7 @@ func ensureInstallMarkerPresent() error {
 	// Otherwise, we're being upgraded from a version of an installed Agent
 	// that didn't use an installation marker file (that is, before v8.8.0).
 	// So create the file now.
-	if err := info.CreateInstallMarker(paths.Top()); err != nil {
+	if err := info.CreateInstallMarker(paths.Top(), utils.CurrentFileOwner()); err != nil {
 		return fmt.Errorf("unable to create installation marker file during upgrade: %w", err)
 	}