support multi stage builds

cmd/executor/cmd/root.go

@@ -32,14 +32,13 @@ import (
 )
 
 var (
-	dockerfilePath              string
-	destination                 string
-	srcContext                  string
-	snapshotMode                string
-	bucket                      string
-	dockerInsecureSkipTLSVerify bool
-	logLevel                    string
-	force                       bool
+	dockerfilePath string
+	destination    string
+	srcContext     string
+	snapshotMode   string
+	bucket         string
+	logLevel       string
+	force          bool
 )
 
 func init() {
@@ -48,7 +47,6 @@ func init() {
 	RootCmd.PersistentFlags().StringVarP(&bucket, "bucket", "b", "", "Name of the GCS bucket from which to access build context as tarball.")
 	RootCmd.PersistentFlags().StringVarP(&destination, "destination", "d", "", "Registry the final image should be pushed to (ex: gcr.io/test/example:latest)")
 	RootCmd.PersistentFlags().StringVarP(&snapshotMode, "snapshotMode", "", "full", "Set this flag to change the file attributes inspected during snapshotting")
-	RootCmd.PersistentFlags().BoolVarP(&dockerInsecureSkipTLSVerify, "insecure-skip-tls-verify", "", false, "Push to insecure registry ignoring TLS verify")
 	RootCmd.PersistentFlags().StringVarP(&logLevel, "verbosity", "v", constants.DefaultLogLevel, "Log level (debug, info, warn, error, fatal, panic")
 	RootCmd.PersistentFlags().BoolVarP(&force, "force", "", false, "Force building outside of a container")
 }
@@ -76,7 +74,12 @@ var RootCmd = &cobra.Command{
 			logrus.Error(err)
 			os.Exit(1)
 		}
-		if err := executor.DoBuild(dockerfilePath, srcContext, destination, snapshotMode, dockerInsecureSkipTLSVerify); err != nil {
+		ref, image, err := executor.DoBuild(dockerfilePath, srcContext, snapshotMode)
+		if err != nil {
+			logrus.Error(err)
+			os.Exit(1)
+		}
+		if err := executor.DoPush(ref, image, destination); err != nil {
 			logrus.Error(err)
 			os.Exit(1)
 		}

integration_tests/dockerfiles/Dockerfile_test_multistage

@@ -0,0 +1,9 @@
+FROM gcr.io/distroless/base:latest
+COPY . .
+
+FROM scratch as second
+ENV foopath context/foo
+COPY --from=0 $foopath context/b* /foo/
+
+FROM gcr.io/distroless/base:latest
+COPY --from=second /foo /foo2

integration_tests/dockerfiles/config_test_multistage.json

@@ -0,0 +1,12 @@
+[
+  {
+    "Image1": "gcr.io/kaniko-test/docker-test-multistage:latest",
+    "Image2": "gcr.io/kaniko-test/kaniko-test-multistage:latest",
+    "DiffType": "File",
+    "Diff": {
+      "Adds": null,
+      "Dels": null,
+      "Mods": null
+    }
+  }
+]

integration_tests/integration_test_yaml.go

@@ -115,14 +115,6 @@ var fileTests = []struct {
 		kanikoContext:  buildcontextPath,
 		repo:           "test-add",
 	},
-	{
-		description:    "test mv add",
-		dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_mv_add",
-		configPath:     "/workspace/integration_tests/dockerfiles/config_test_mv_add.json",
-		dockerContext:  buildcontextPath,
-		kanikoContext:  buildcontextPath,
-		repo:           "test-mv-add",
-	},
 	{
 		description:    "test registry",
 		dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_registry",
@@ -147,6 +139,14 @@ var fileTests = []struct {
 		kanikoContext:  buildcontextPath,
 		repo:           "test-scratch",
 	},
+	{
+		description:    "test multistage",
+		dockerfilePath: "/workspace/integration_tests/dockerfiles/Dockerfile_test_multistage",
+		configPath:     "/workspace/integration_tests/dockerfiles/config_test_multistage.json",
+		dockerContext:  buildcontextPath,
+		kanikoContext:  buildcontextPath,
+		repo:           "test-multistage",
+	},
 }
 
 var structureTests = []struct {
@@ -288,7 +288,7 @@ func main() {
 		}
 		compareOutputs := step{
 			Name: ubuntuImage,
-			Args: []string{"cmp", "-b", test.configPath, containerDiffOutputFile},
+			Args: []string{"cmp", test.configPath, containerDiffOutputFile},
 		}
 
 		y.Steps = append(y.Steps, dockerBuild, kaniko, pullKanikoImage, containerDiff, catContainerDiffOutput, compareOutputs)

pkg/commands/copy.go

@@ -17,6 +17,7 @@ limitations under the License.
 package commands
 
 import (
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
 	"os"
 	"path/filepath"
 	"strings"
@@ -40,6 +41,10 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config) error {
 	logrus.Infof("cmd: copy %s", srcs)
 	logrus.Infof("dest: %s", dest)
 
+	// Resolve from
+	if c.cmd.From != "" {
+		c.buildcontext = filepath.Join(constants.BuildContextDir, c.cmd.From)
+	}
 	// First, resolve any environment replacement
 	resolvedEnvs, err := util.ResolveEnvironmentReplacementList(c.cmd.SourcesAndDest, config.Env, true)
 	if err != nil {
@@ -58,11 +63,19 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config) error {
 		if err != nil {
 			return err
 		}
+		cwd := config.WorkingDir
+		if cwd == "" {
+			cwd = constants.RootDir
+		}
 		destPath, err := util.DestinationFilepath(src, dest, config.WorkingDir)
 		if err != nil {
 			return err
 		}
 		if fi.IsDir() {
+			if !filepath.IsAbs(dest) {
+				// we need to add '/' to the end to indicate the destination is a directory
+				dest = filepath.Join(cwd, dest) + "/"
+			}
 			if err := util.CopyDir(fullPath, dest); err != nil {
 				return err
 			}

pkg/commands/env.go

@@ -29,6 +29,12 @@ type EnvCommand struct {
 	cmd *instructions.EnvCommand
 }
 
+func NewEnvCommand(cmd *instructions.EnvCommand) EnvCommand {
+	return EnvCommand{
+		cmd: cmd,
+	}
+}
+
 func (e *EnvCommand) ExecuteCommand(config *v1.Config) error {
 	logrus.Info("cmd: ENV")
 	newEnvs := e.cmd.Env

pkg/dockerfile/dockerfile.go

@@ -18,10 +18,20 @@ package dockerfile
 
 import (
 	"bytes"
-	"strings"
-
+	"github.com/GoogleContainerTools/kaniko/pkg/commands"
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
 	"github.com/docker/docker/builder/dockerfile/instructions"
 	"github.com/docker/docker/builder/dockerfile/parser"
+	"github.com/google/go-containerregistry/authn"
+	"github.com/google/go-containerregistry/name"
+	"github.com/google/go-containerregistry/v1"
+	"github.com/google/go-containerregistry/v1/empty"
+	"github.com/google/go-containerregistry/v1/remote"
+	"net/http"
+	"path/filepath"
+	"strconv"
+	"strings"
 )
 
 // Parse parses the contents of a Dockerfile and returns a list of commands
@@ -37,6 +47,25 @@ func Parse(b []byte) ([]instructions.Stage, error) {
 	return stages, err
 }
 
+// ResolveStages resolves any calls to previous stages with names to indices
+// Ex. --from=second_stage should be --from=1 for easier processing later on
+func ResolveStages(stages []instructions.Stage) {
+	nameToIndex := make(map[string]string)
+	for i, stage := range stages {
+		index := strconv.Itoa(i)
+		nameToIndex[stage.Name] = index
+		nameToIndex[index] = index
+		for _, cmd := range stage.Commands {
+			switch c := cmd.(type) {
+			case *instructions.CopyCommand:
+				if c.From != "" {
+					c.From = nameToIndex[c.From]
+				}
+			}
+		}
+	}
+}
+
 // ParseCommands parses an array of commands into an array of instructions.Command; used for onbuild
 func ParseCommands(cmdArray []string) ([]instructions.Command, error) {
 	var cmds []instructions.Command
@@ -54,3 +83,66 @@ func ParseCommands(cmdArray []string) ([]instructions.Command, error) {
 	}
 	return cmds, nil
 }
+
+// Dependencies returns a list of files in this stage that will be needed in later stages
+func Dependencies(index int, stages []instructions.Stage) ([]string, error) {
+	var dependencies []string
+	for stageIndex, stage := range stages {
+		if stageIndex <= index {
+			continue
+		}
+		var sourceImage v1.Image
+		if stage.BaseName == constants.NoBaseImage {
+			sourceImage = empty.Image
+		} else {
+			// Initialize source image
+			ref, err := name.ParseReference(stage.BaseName, name.WeakValidation)
+			if err != nil {
+				return nil, err
+
+			}
+			auth, err := authn.DefaultKeychain.Resolve(ref.Context().Registry)
+			if err != nil {
+				return nil, err
+			}
+			sourceImage, err = remote.Image(ref, auth, http.DefaultTransport)
+			if err != nil {
+				return nil, err
+			}
+		}
+		imageConfig, err := sourceImage.ConfigFile()
+		if err != nil {
+			return nil, err
+		}
+		for _, cmd := range stage.Commands {
+			switch c := cmd.(type) {
+			case *instructions.EnvCommand:
+				envCommand := commands.NewEnvCommand(c)
+				if err := envCommand.ExecuteCommand(&imageConfig.Config); err != nil {
+					return nil, err
+				}
+			case *instructions.CopyCommand:
+				if c.From != strconv.Itoa(index) {
+					continue
+				}
+				// First, resolve any environment replacement
+				resolvedEnvs, err := util.ResolveEnvironmentReplacementList(c.SourcesAndDest, imageConfig.Config.Env, true)
+				if err != nil {
+					return nil, err
+				}
+				// Resolve wildcards and get a list of resolved sources
+				srcs, err := util.ResolveSources(resolvedEnvs, constants.RootDir)
+				if err != nil {
+					return nil, err
+				}
+				for index, src := range srcs {
+					if !filepath.IsAbs(src) {
+						srcs[index] = filepath.Join(constants.RootDir, src)
+					}
+				}
+				dependencies = append(dependencies, srcs...)
+			}
+		}
+	}
+	return dependencies, nil
+}

pkg/dockerfile/dockerfile_test.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockerfile
+
+import (
+	"fmt"
+	"github.com/GoogleContainerTools/kaniko/testutil"
+	"github.com/docker/docker/builder/dockerfile/instructions"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+)
+
+var dockerfile = `
+FROM scratch
+RUN echo hi > /hi
+
+FROM scratch AS second
+COPY --from=0 /hi /hi2
+
+FROM scratch
+COPY --from=second /hi2 /hi3
+`
+
+func Test_ResolveStages(t *testing.T) {
+	stages, err := Parse([]byte(dockerfile))
+	if err != nil {
+		t.Fatal(err)
+	}
+	ResolveStages(stages)
+	for index, stage := range stages {
+		if index == 0 {
+			continue
+		}
+		copyCmd := stage.Commands[0].(*instructions.CopyCommand)
+		expectedStage := strconv.Itoa(index - 1)
+		if copyCmd.From != expectedStage {
+			t.Fatalf("unexpected copy command: %s resolved to stage %s, expected %s", copyCmd.String(), copyCmd.From, expectedStage)
+		}
+	}
+}
+
+func Test_Dependencies(t *testing.T) {
+	testDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	helloPath := filepath.Join(testDir, "hello")
+	if err := os.Mkdir(helloPath, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	dockerfile := fmt.Sprintf(`
+	FROM scratch
+	COPY %s %s
+	
+	FROM scratch AS second
+	ENV hienv %s
+	COPY a b
+	COPY --from=0 /$hienv %s /hi2/
+	`, helloPath, helloPath, helloPath, testDir)
+
+	stages, err := Parse([]byte(dockerfile))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedDependencies := [][]string{
+		{
+			helloPath,
+			testDir,
+		},
+		nil,
+	}
+
+	for index := range stages {
+		actualDeps, err := Dependencies(index, stages)
+		testutil.CheckErrorAndDeepEqual(t, false, err, expectedDependencies[index], actualDeps)
+	}
+}