Select entrypoint command based on runtime platform

This fixes a long-standing bug affecting heterogenous clusters, where
the controller's platform would be used to lookup the image's
entrypoint, instead of the platform of the node where the workload would
eventually run.

With this change, the controller looks up _all_ the image's entrypoints
and passes them to the entrypoint binary on the node, where it uses its
current runtime platform to lookup the correct entrypoint to execute.

This has the added benefit that we can now pass the entire image@digest
of the multi-platform image down to the Pod, instead of the
(controller's) platform-specific image. This has benefits for scenarios
where Pods may be blocked from running unsigned/untrusted images, since
it might be the multi-platform image index that's signed/trusted, and
not any particular platform-specific constituent image.

cmd/entrypoint/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"encoding/json"
 	"flag"
 	"log"
 	"os"
@@ -25,6 +26,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/containerd/containerd/platforms"
 	"github.com/tektoncd/pipeline/cmd/entrypoint/subcommands"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline"
 	"github.com/tektoncd/pipeline/pkg/credentials"
@@ -99,13 +101,30 @@ func main() {
 		}
 	}
 
+	var cmd []string
+	if *ep != "" {
+		cmd = []string{*ep}
+	} else {
+		env := os.Getenv("TEKTON_PLATFORM_COMMANDS")
+		var cmds map[string][]string
+		if err := json.Unmarshal([]byte(env), &cmds); err != nil {
+			log.Fatal(err)
+		}
+		plat := platforms.DefaultString()
+
+		var found bool
+		cmd, found = cmds[plat]
+		if !found {
+			log.Fatalf("could not find command for platform %q", plat)
+		}
+	}
+
 	e := entrypoint.Entrypointer{
-		Entrypoint:          *ep,
+		Command:             append(cmd, flag.Args()...),
 		WaitFiles:           strings.Split(*waitFiles, ","),
 		WaitFileContent:     *waitFileContent,
 		PostFile:            *postFile,
 		TerminationPath:     *terminationPath,
-		Args:                flag.Args(),
 		Waiter:              &realWaiter{waitPollingInterval: defaultWaitPollingInterval, breakpointOnFailure: *breakpointOnFailure},
 		Runner:              &realRunner{},
 		PostWriter:          &realPostWriter{},

examples/v1beta1/taskruns/entrypoint-resolution.yaml

@@ -0,0 +1,49 @@
+apiVersion: tekton.dev/v1beta1
+kind: TaskRun
+metadata:
+  name: entrypoint-resolution
+spec:
+  taskSpec:
+    steps:
+    # Multi-arch image with no command defined. We should look up the command
+    # for each platform-specific image and pass it to the Pod, which selects
+    # the right command at runtime based on the node's runtime platform.
+    - image: gcr.io/tekton-nightly/github.com/tektoncd/pipeline/cmd/nop
+
+    # Single-platform image with no command defined, but with args. We'll look
+    # up the commands and pass it to the entrypoint binary via env var, then
+    # append the specified args.
+    - image: ubuntu
+      args: ['-c', 'echo', 'hello']
+
+    # Multi-arch image, but since we specify `script` we don't need to look it
+    # up and pass it down.
+    - image: ubuntu
+      script: echo hello
+
+    # Multi-arch image, but since we specify `command` and `args` we don't
+    # need to look it up and pass it down.
+    - image: ubuntu
+      command: ['sh', '-c']
+      args: ['echo hello']
+
+    # Single-platform image with no command defined. We should look up the one
+    # and only command value and pass it to the Pod.
+    - image: amd64/ubuntu
+
+    # Single-platform image with no command defined, but with args. We'll look
+    # up the one command and pass it to the entrypoint binary, then append the
+    # specified args.
+    - image: amd64/ubuntu
+      args: ['-c', 'echo', 'hello']
+
+    # Single-platform image, but since we specify `script` we don't need to
+    # look it up and pass it down.
+    - image: amd64/ubuntu
+      script: echo hello
+
+    # Single-platform image, but since we specify `command` and `args` we
+    # don't need to look it up and pass it down.
+    - image: amd64/ubuntu
+      command: ['sh', '-c']
+      args: ['echo hello']

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/cloudevents/sdk-go/v2 v2.5.0
+	github.com/containerd/containerd v1.5.2
 	github.com/docker/cli v20.10.8+incompatible // indirect
 	github.com/docker/docker v20.10.8+incompatible // indirect
 	github.com/emicklei/go-restful v2.15.0+incompatible // indirect
@@ -15,9 +16,9 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/googleapis/gnostic v0.5.3 // indirect
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/hashicorp/golang-lru v0.5.4
 	github.com/jenkins-x/go-scm v1.10.10
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/opencontainers/image-spec v1.0.3-0.20211202222133-eacdcc10569b
 	github.com/pkg/errors v0.9.1
 	github.com/tektoncd/plumbing v0.0.0-20211012143332-c7cc43d9bc0c
 	go.opencensus.io v0.23.0

pkg/entrypoint/entrypointer.go

@@ -45,10 +45,9 @@ const (
 // Entrypointer holds fields for running commands with redirected
 // entrypoints.
 type Entrypointer struct {
-	// Entrypoint is the original specified entrypoint, if any.
-	Entrypoint string
-	// Args are the original specified args, if any.
-	Args []string
+	// Command is the original specified command and args.
+	Command []string
+
 	// WaitFiles is the set of files to wait for. If empty, execution
 	// begins immediately.
 	WaitFiles []string
@@ -141,10 +140,6 @@ func (e Entrypointer) Go() error {
 		}
 	}
 
-	if e.Entrypoint != "" {
-		e.Args = append([]string{e.Entrypoint}, e.Args...)
-	}
-
 	output = append(output, v1beta1.PipelineResourceResult{
 		Key:        "StartedAt",
 		Value:      time.Now().Format(timeFormat),
@@ -163,7 +158,7 @@ func (e Entrypointer) Go() error {
 			ctx, cancel = context.WithTimeout(ctx, *e.Timeout)
 			defer cancel()
 		}
-		err = e.Runner.Run(ctx, e.Args...)
+		err = e.Runner.Run(ctx, e.Command...)
 		if err == context.DeadlineExceeded {
 			output = append(output, v1beta1.PipelineResourceResult{
 				Key:        "Reason",

pkg/entrypoint/entrypointer_test.go

@@ -94,10 +94,9 @@ func TestEntrypointerFailures(t *testing.T) {
 				defer os.Remove(terminationFile.Name())
 			}
 			err := Entrypointer{
-				Entrypoint:      "echo",
+				Command:         []string{"echo", "some", "args"},
 				WaitFiles:       c.waitFiles,
 				PostFile:        c.postFile,
-				Args:            []string{"some", "args"},
 				Waiter:          fw,
 				Runner:          fr,
 				PostWriter:      fpw,
@@ -175,10 +174,9 @@ func TestEntrypointer(t *testing.T) {
 				defer os.Remove(terminationFile.Name())
 			}
 			err := Entrypointer{
-				Entrypoint:          c.entrypoint,
+				Command:             append([]string{c.entrypoint}, c.args...),
 				WaitFiles:           c.waitFiles,
 				PostFile:            c.postFile,
-				Args:                c.args,
 				Waiter:              fw,
 				Runner:              fr,
 				PostWriter:          fpw,
@@ -203,10 +201,7 @@ func TestEntrypointer(t *testing.T) {
 				t.Errorf("Waited for file when not required")
 			}
 
-			wantArgs := c.args
-			if c.entrypoint != "" {
-				wantArgs = append([]string{c.entrypoint}, c.args...)
-			}
+			wantArgs := append([]string{c.entrypoint}, c.args...)
 			if len(wantArgs) != 0 {
 				if fr.args == nil {
 					t.Error("Wanted command to be run, got nil")
@@ -324,10 +319,9 @@ func TestEntrypointer_OnError(t *testing.T) {
 				defer os.Remove(terminationFile.Name())
 			}
 			err := Entrypointer{
-				Entrypoint:      "echo",
+				Command:         []string{"echo", "some", "args"},
 				WaitFiles:       []string{},
 				PostFile:        c.postFile,
-				Args:            []string{"some", "args"},
 				Waiter:          &fakeWaiter{},
 				Runner:          c.runner,
 				PostWriter:      fpw,

pkg/pod/entrypoint.go

@@ -147,15 +147,6 @@ func orderContainers(commonExtraEntrypointArgs []string, steps []corev1.Containe
 			argsForEntrypoint = append(argsForEntrypoint, resultArgument(steps, taskSpec.Results)...)
 		}
 
-		cmd, args := s.Command, s.Args
-		if len(cmd) == 0 {
-			return nil, fmt.Errorf("Step %d did not specify command", i)
-		}
-		if len(cmd) > 1 {
-			args = append(cmd[1:], args...)
-			cmd = []string{cmd[0]}
-		}
-
 		if breakpointConfig != nil && len(breakpointConfig.Breakpoint) > 0 {
 			breakpoints := breakpointConfig.Breakpoint
 			for _, b := range breakpoints {
@@ -166,7 +157,14 @@ func orderContainers(commonExtraEntrypointArgs []string, steps []corev1.Containe
 			}
 		}
 
-		argsForEntrypoint = append(argsForEntrypoint, "-entrypoint", cmd[0], "--")
+		cmd, args := s.Command, s.Args
+		if len(cmd) > 0 {
+			argsForEntrypoint = append(argsForEntrypoint, "-entrypoint", cmd[0])
+		}
+		if len(cmd) > 1 {
+			args = append(cmd[1:], args...)
+		}
+		argsForEntrypoint = append(argsForEntrypoint, "--")
 		argsForEntrypoint = append(argsForEntrypoint, args...)
 
 		steps[i].Command = []string{entrypointBinary}

pkg/pod/entrypoint_lookup.go

@@ -18,7 +18,7 @@ package pod
 
 import (
 	"context"
-	"fmt"
+	"encoding/json"
 
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -31,10 +31,18 @@ type EntrypointCache interface {
 	// Get the Image data for the given image reference. If the value is
 	// not found in the cache, it will be fetched from the image registry,
 	// possibly using K8s service account imagePullSecrets.
-	Get(ctx context.Context, ref name.Reference, namespace, serviceAccountName string) (v1.Image, error)
-	// Set updates the cache with a new digest->Image mapping. This will avoid a
-	// remote registry lookup next time Get is called.
-	Set(digest name.Digest, img v1.Image)
+	//
+	// It also returns the digest associated with the given reference. If
+	// the reference referred to an index, the returned digest will be the
+	// index's digest, not any platform-specific image contained by the
+	// index.
+	Get(ctx context.Context, ref name.Reference, namespace, serviceAccountName string) (*imageData, error)
+}
+
+// imageData contains information looked up about an image or multi-platform image index.
+type imageData struct {
+	digest   v1.Hash
+	commands map[string][]string // map of platform -> []command
 }
 
 // resolveEntrypoints looks up container image ENTRYPOINTs for all steps that
@@ -43,70 +51,57 @@ type EntrypointCache interface {
 // Images that are not specified by digest will be specified by digest after
 // lookup in the resulting list of containers.
 func resolveEntrypoints(ctx context.Context, cache EntrypointCache, namespace, serviceAccountName string, steps []corev1.Container) ([]corev1.Container, error) {
-	// Keep a local cache of name->image lookups, just for the scope of
+	// Keep a local cache of name->imageData lookups, just for the scope of
 	// resolving this set of steps. If the image is pushed to before the
-	// next run, we need to resolve its digest and entrypoint again, but we
+	// next run, we need to resolve its digest and commands again, but we
 	// can skip lookups while resolving the same TaskRun.
-	localCache := map[name.Reference]v1.Image{}
+	localCache := map[name.Reference]imageData{}
 	for i, s := range steps {
-		if len(s.Command) != 0 {
-			// Nothing to resolve.
+
+		// If the command is already specified, there's nothing to resolve.
+		if len(s.Command) > 0 {
 			continue
 		}
 
-		origRef, err := name.ParseReference(s.Image, name.WeakValidation)
+		ref, err := name.ParseReference(s.Image, name.WeakValidation)
 		if err != nil {
 			return nil, err
 		}
-		var img v1.Image
-		if cimg, found := localCache[origRef]; found {
-			img = cimg
+		var id imageData
+		if cid, found := localCache[ref]; found {
+			id = cid
 		} else {
-			// Look it up in the cache. If it's not found in the
-			// cache, it will be resolved from the registry.
-			img, err = cache.Get(ctx, origRef, namespace, serviceAccountName)
+			// Look it up for real.
+			lid, err := cache.Get(ctx, ref, namespace, serviceAccountName)
 			if err != nil {
 				return nil, err
 			}
-			// Cache it locally in case another step specifies the same image.
-			localCache[origRef] = img
-		}
+			id = *lid
 
-		ep, digest, err := imageData(origRef, img)
-		if err != nil {
-			return nil, err
+			// Cache it locally in case another step in this task specifies the same image.
+			localCache[ref] = *lid
 		}
 
-		cache.Set(digest, img) // Cache the lookup for next time this image is looked up by digest.
+		// Resolve the original reference to a reference by digest.
+		steps[i].Image = ref.Context().Digest(id.digest.String()).String()
 
-		steps[i].Image = digest.String() // Specify image by digest, since we know it now.
-		steps[i].Command = ep            // Specify the command explicitly.
+		if len(id.commands) == 1 {
+			// Promote the single found command to the step's command.
+			for _, v := range id.commands {
+				steps[i].Command = v
+				break
+			}
+		} else {
+			// Encode the map of platform->command to JSON and pass it via env var.
+			b, err := json.Marshal(id.commands)
+			if err != nil {
+				return nil, err
+			}
+			steps[i].Env = append(steps[i].Env, corev1.EnvVar{
+				Name:  "TEKTON_PLATFORM_COMMANDS",
+				Value: string(b),
+			})
+		}
 	}
 	return steps, nil
 }
-
-// imageData pulls the entrypoint from the image, and returns the given
-// original reference, with image digest resolved.
-func imageData(ref name.Reference, img v1.Image) ([]string, name.Digest, error) {
-	digest, err := img.Digest()
-	if err != nil {
-		return nil, name.Digest{}, fmt.Errorf("error getting image digest: %v", err)
-	}
-	cfg, err := img.ConfigFile()
-	if err != nil {
-		return nil, name.Digest{}, fmt.Errorf("error getting image config: %v", err)
-	}
-
-	// Entrypoint can be specified in either .Config.Entrypoint or
-	// .Config.Cmd.
-	ep := cfg.Config.Entrypoint
-	if len(ep) == 0 {
-		ep = cfg.Config.Cmd
-	}
-
-	d, err := name.NewDigest(ref.Context().String()+"@"+digest.String(), name.WeakValidation)
-	if err != nil {
-		return nil, name.Digest{}, fmt.Errorf("error constructing resulting digest: %v", err)
-	}
-	return ep, d, nil
-}