drivers/docker+exec+java: disable net_raw capability by default

CHANGELOG.md

@@ -15,6 +15,9 @@ __BACKWARDS INCOMPATIBILITIES:__
  * csi: The `attachment_mode` and `access_mode` field are required for `volume` blocks in job specifications. Registering a volume requires at least one `capability` block with the `attachment_mode` and `access_mode` fields set. [[GH-10330](https://github.com/hashicorp/nomad/issues/10330)]
  * licensing: Enterprise licenses are no longer stored in raft or synced between servers. Loading the Enterprise license from disk or environment is required. The `nomad license put` command has been removed. [[GH-10458](https://github.com/hashicorp/nomad/issues/10458)]
 
+SECURITY:
+ * drivers/docker+exec+java: Disable `CAP_NET_RAW` linux capability by default to prevent ARP spoofing. CVE-2021-32575 [GH-10568](https://github.com/hashicorp/nomad/issues/10568)
+
 IMPROVEMENTS:
  * api: Added an API endpoint for fuzzy search queries [[GH-10184](https://github.com/hashicorp/nomad/pull/10184)]
  * api: Removed unimplemented `CSIVolumes.PluginList` API. [[GH-10158](https://github.com/hashicorp/nomad/issues/10158)]
@@ -72,7 +75,7 @@ BUG FIXES:
  * server: Fixed a panic that may arise on submission of jobs containing invalid service checks [[GH-10154](https://github.com/hashicorp/nomad/issues/10154)]
  * ui: Fixed the rendering of interstitial components shown after processing a dynamic application sizing recommendation. [[GH-10094](https://github.com/hashicorp/nomad/pull/10094)]
 
-## 1.0.5 (Unreleased)
+## 1.0.6 (Unreleased)
 
 BUG FIXES:
  * core (Enterprise): Update licensing library to v0.0.11 to include race condition fix. [[GH-10253](https://github.com/hashicorp/nomad/issues/10253)]
@@ -101,6 +104,11 @@ BUG FIXES:
  * server: Fixed a panic that may arise on submission of jobs containing invalid service checks [[GH-10154](https://github.com/hashicorp/nomad/issues/10154)]
  * ui: Fixed the rendering of interstitial components shown after processing a dynamic application sizing recommendation. [[GH-10094](https://github.com/hashicorp/nomad/pull/10094)]
 
+## 1.0.5 (May 11, 2021)
+
+SECURITY:
+ * drivers/docker+exec+java: Disable `CAP_NET_RAW` linux capability by default to prevent ARP spoofing. CVE-2021-32575 [GH-10568](https://github.com/hashicorp/nomad/issues/10568)
+
 ## 1.0.4 (February 24, 2021)
 
 FEATURES:
@@ -277,6 +285,11 @@ BUG FIXES:
  * ui: Fixed a bug in the volume status page where read allocations and write allocations were not displayed. [[GH-9377](https://github.com/hashicorp/nomad/issues/9377)]
  * ui: Fixed a bug in the CSI volume and plugin status pages where plugins that don't require controllers were shown as unhealthy. [[GH-9416](https://github.com/hashicorp/nomad/issues/9416)]
 
+## 0.12.12 (May 11, 2021)
+
+SECURITY:
+ * drivers/docker+exec+java: Disable `CAP_NET_RAW` linux capability by default to prevent ARP spoofing. CVE-2021-32575 [GH-10568](https://github.com/hashicorp/nomad/issues/10568)
+
 ## 0.12.11 (March 18, 2021)
 
 BUG FIXES:

drivers/docker/config.go

@@ -36,17 +36,41 @@ const (
 	// it is timed out.
 	dockerTimeout = 5 * time.Minute
 
-	// dockerBasicCaps is comma-separated list of Linux capabilities that are
-	// allowed by docker by default, as documented in
-	// https://docs.docker.com/engine/reference/run/#block-io-bandwidth-blkio-constraint
-	dockerBasicCaps = "CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID," +
-		"SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE"
-
 	// dockerAuthHelperPrefix is the prefix to attach to the credential helper
 	// and should be found in the $PATH. Example: ${prefix-}${helper-name}
 	dockerAuthHelperPrefix = "docker-credential-"
 )
 
+// nomadDefaultCaps is the subset of dockerDefaultCaps that Nomad enables by
+// default and is used to compute the set of capabilities to add/drop given
+// docker driver configuration.
+func nomadDefaultCaps() []string {
+	return []string{
+		"AUDIT_WRITE",
+		"CHOWN",
+		"DAC_OVERRIDE",
+		"FOWNER",
+		"FSETID",
+		"KILL",
+		"MKNOD",
+		"NET_BIND_SERVICE",
+		"SETFCAP",
+		"SETGID",
+		"SETPCAP",
+		"SETUID",
+		"SYS_CHROOT",
+	}
+}
+
+// dockerDefaultCaps is a list of Linux capabilities enabled by docker by default
+// and is used to compute the set of capabilities to add/drop given docker driver
+// configuration, as well as Nomad built-in limitations.
+//
+// https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities
+func dockerDefaultCaps() []string {
+	return append(nomadDefaultCaps(), "NET_RAW")
+}
+
 func PluginLoader(opts map[string]string) (map[string]interface{}, error) {
 	conf := map[string]interface{}{}
 	if v, ok := opts["docker.endpoint"]; ok {
@@ -263,7 +287,7 @@ var (
 		"allow_privileged": hclspec.NewAttr("allow_privileged", "bool", false),
 		"allow_caps": hclspec.NewDefault(
 			hclspec.NewAttr("allow_caps", "list(string)", false),
-			hclspec.NewLiteral(`["CHOWN","DAC_OVERRIDE","FSETID","FOWNER","MKNOD","NET_RAW","SETGID","SETUID","SETFCAP","SETPCAP","NET_BIND_SERVICE","SYS_CHROOT","KILL","AUDIT_WRITE"]`),
+			hclspec.NewLiteral(`["CHOWN","DAC_OVERRIDE","FSETID","FOWNER","MKNOD","SETGID","SETUID","SETFCAP","SETPCAP","NET_BIND_SERVICE","SYS_CHROOT","KILL","AUDIT_WRITE"]`),
 		),
 		"nvidia_runtime": hclspec.NewDefault(
 			hclspec.NewAttr("nvidia_runtime", "string", false),

drivers/docker/driver.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -23,7 +24,9 @@ import (
 	"github.com/hashicorp/nomad/client/taskenv"
 	"github.com/hashicorp/nomad/drivers/docker/docklog"
 	"github.com/hashicorp/nomad/drivers/shared/eventer"
+	"github.com/hashicorp/nomad/drivers/shared/executor"
 	"github.com/hashicorp/nomad/drivers/shared/resolvconf"
+	"github.com/hashicorp/nomad/helper"
 	nstructs "github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/plugins/base"
 	"github.com/hashicorp/nomad/plugins/drivers"
@@ -909,38 +912,12 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
 	}
 	hostConfig.Privileged = driverConfig.Privileged
 
-	// set capabilities
-	hostCapsWhitelistConfig := d.config.AllowCaps
-	hostCapsWhitelist := make(map[string]struct{})
-	for _, cap := range hostCapsWhitelistConfig {
-		cap = strings.ToLower(strings.TrimSpace(cap))
-		hostCapsWhitelist[cap] = struct{}{}
-	}
-
-	if _, ok := hostCapsWhitelist["all"]; !ok {
-		effectiveCaps, err := tweakCapabilities(
-			strings.Split(dockerBasicCaps, ","),
-			driverConfig.CapAdd,
-			driverConfig.CapDrop,
-		)
-		if err != nil {
-			return c, err
-		}
-		var missingCaps []string
-		for _, cap := range effectiveCaps {
-			cap = strings.ToLower(cap)
-			if _, ok := hostCapsWhitelist[cap]; !ok {
-				missingCaps = append(missingCaps, cap)
-			}
-		}
-		if len(missingCaps) > 0 {
-			return c, fmt.Errorf("Docker driver doesn't have the following caps allowlisted on this Nomad agent: %s", missingCaps)
-		}
+	// set add/drop capabilities
+	hostConfig.CapAdd, hostConfig.CapDrop, err = d.getCaps(driverConfig)
+	if err != nil {
+		return c, err
 	}
 
-	hostConfig.CapAdd = driverConfig.CapAdd
-	hostConfig.CapDrop = driverConfig.CapDrop
-
 	// set SHM size
 	if driverConfig.ShmSize != 0 {
 		hostConfig.ShmSize = driverConfig.ShmSize
@@ -1207,6 +1184,119 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
 	}, nil
 }
 
+// getCaps computes the capabilities to supply to the --add-cap and --drop-cap
+// options to the docker driver, which override the default capabilities enabled
+// by docker itself.
+func (d *Driver) getCaps(taskConfig *TaskConfig) ([]string, []string, error) {
+
+	// capabilities allowable by client docker plugin configuration
+	allowCaps := expandAllowCaps(d.config.AllowCaps)
+
+	// capabilities the task docker config is asking for based on the default
+	// capabilities allowable by nomad
+	desiredCaps, err := tweakCapabilities(nomadDefaultCaps(), taskConfig.CapAdd, taskConfig.CapDrop)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// capabilities the task is requesting that are NOT allowed by the docker plugin
+	if missing := missingCaps(allowCaps, desiredCaps); len(missing) > 0 {
+		return nil, nil, fmt.Errorf("Docker driver does not have the following caps allow-listed on this Nomad agent: %s", missing)
+	}
+
+	// capabilities that should be dropped relative to the docker default capabilities
+	dropCaps := capDrops(taskConfig.CapDrop, allowCaps)
+
+	return taskConfig.CapAdd, dropCaps, nil
+}
+
+// capDrops will compute the total dropped capabilities set
+//
+// {task cap_drop} U ({docker defaults} \ {driver allow caps})
+func capDrops(dropCaps []string, allowCaps []string) []string {
+	dropSet := make(map[string]struct{})
+
+	for _, c := range normalizeCaps(dropCaps) {
+		dropSet[c] = struct{}{}
+	}
+
+	// if dropCaps includes ALL, no need to iterate every capability
+	if _, exists := dropSet["ALL"]; exists {
+		return []string{"ALL"}
+	}
+
+	dockerDefaults := helper.SliceStringToSet(normalizeCaps(dockerDefaultCaps()))
+	allowedCaps := helper.SliceStringToSet(normalizeCaps(allowCaps))
+
+	// find the docker default caps not in allowed caps
+	for dCap := range dockerDefaults {
+		if _, exists := allowedCaps[dCap]; !exists {
+			dropSet[dCap] = struct{}{}
+		}
+	}
+
+	drops := make([]string, 0, len(dropSet))
+	for c := range dropSet {
+		drops = append(drops, c)
+	}
+	sort.Strings(drops)
+	return drops
+}
+
+// expandAllowCaps returns the normalized set of allowable capabilities set
+// for the docker plugin configuration.
+func expandAllowCaps(allowCaps []string) []string {
+	if len(allowCaps) == 0 {
+		return nil
+	}
+
+	set := make(map[string]struct{}, len(allowCaps))
+
+	for _, rawCap := range allowCaps {
+		capability := strings.ToUpper(rawCap)
+		if capability == "ALL" {
+			for _, defCap := range normalizeCaps(executor.SupportedCaps(true)) {
+				set[defCap] = struct{}{}
+			}
+		} else {
+			set[capability] = struct{}{}
+		}
+	}
+
+	result := make([]string, 0, len(set))
+	for capability := range set {
+		result = append(result, capability)
+	}
+	sort.Strings(result)
+	return result
+}
+
+// missingCaps returns the set of elements in desired that are not present in
+// allowed. The elements in desired are first upper-cased before comparison.
+// The elements in allowed are assumed to be upper-cased.
+func missingCaps(allowed, desired []string) []string {
+	_, missing := helper.SliceStringIsSubset(allowed, normalizeCaps(desired))
+	sort.Strings(missing)
+	return missing
+}
+
+// normalizeCaps returns a copy of caps with duplicate elements removed and all
+// elements upper-cased.
+func normalizeCaps(caps []string) []string {
+	set := make(map[string]struct{}, len(caps))
+	for _, c := range caps {
+		normal := strings.TrimPrefix(strings.ToUpper(c), "CAP_")
+		set[strings.ToUpper(normal)] = struct{}{}
+	}
+
+	result := make([]string, 0, len(set))
+	for c := range set {
+		result = append(result, c)
+	}
+	sort.Strings(result)
+	return result
+}
+
 func (d *Driver) toDockerMount(m *DockerMount, task *drivers.TaskConfig) (*docker.HostMount, error) {
 	hm, err := m.toDockerHostMount()
 	if err != nil {

drivers/docker/driver_default.go

@@ -27,5 +27,6 @@ func tweakCapabilities(basics, adds, drops []string) ([]string, error) {
 	for i, cap := range effectiveCaps {
 		effectiveCaps[i] = cap[len("CAP_"):]
 	}
+
 	return effectiveCaps, nil
 }