Merge pull request appc#237 from dgonyeo/annotate-manifest-hash

Annotate manifest hash

lib/common/common.go

@@ -48,6 +48,7 @@ const (
 	AppcDockerParentImageID = "appc.io/docker/parentimageid"
 	AppcDockerEntrypoint    = "appc.io/docker/entrypoint"
 	AppcDockerCmd           = "appc.io/docker/cmd"
+	AppcDockerManifestHash  = "appc.io/docker/manifesthash"
 )
 
 const defaultTag = "latest"

lib/docker2aci.go

@@ -42,10 +42,11 @@ import (
 // CommonConfig represents the shared configuration options for converting
 // Docker images.
 type CommonConfig struct {
-	Squash      bool               // squash the layers in one file
-	OutputDir   string             // where to put the resulting ACI
-	TmpDir      string             // directory to use for temporary files
-	Compression common.Compression // which compression to use for the resulting file(s)
+	Squash                bool               // squash the layers in one file
+	OutputDir             string             // where to put the resulting ACI
+	TmpDir                string             // directory to use for temporary files
+	Compression           common.Compression // which compression to use for the resulting file(s)
+	CurrentManifestHashes []string           // any manifest hashes the caller already has
 
 	Info  log.Logger
 	Debug log.Logger
@@ -141,10 +142,15 @@ type converter struct {
 
 func (c *converter) convert() ([]string, error) {
 	c.config.Debug.Println("Getting image info...")
-	ancestry, parsedDockerURL, err := c.backend.GetImageInfo(c.dockerURL)
+	ancestry, manhash, parsedDockerURL, err := c.backend.GetImageInfo(c.dockerURL)
 	if err != nil {
 		return nil, err
 	}
+	for _, h := range c.config.CurrentManifestHashes {
+		if manhash == h {
+			return nil, nil
+		}
+	}
 
 	layersOutputDir := c.config.OutputDir
 	if c.config.Squash {
@@ -163,7 +169,7 @@ func (c *converter) convert() ([]string, error) {
 		layerCompression = common.NoCompression
 	}
 
-	aciLayerPaths, aciManifests, err := c.backend.BuildACI(ancestry, parsedDockerURL, layersOutputDir, c.config.TmpDir, layerCompression)
+	aciLayerPaths, aciManifests, err := c.backend.BuildACI(ancestry, manhash, parsedDockerURL, layersOutputDir, c.config.TmpDir, layerCompression)
 	if err != nil {
 		return nil, err
 	}

lib/internal/backend/file/file.go

@@ -53,15 +53,21 @@ func NewFileBackend(file *os.File, debug, info log.Logger) *FileBackend {
 	}
 }
 
-func (lb *FileBackend) GetImageInfo(dockerURL string) ([]string, *common.ParsedDockerURL, error) {
+// GetImageInfo, given the url for a docker image, will return the
+// following:
+// - []string: an ordered list of all layer hashes
+// - string: a unique identifier for this image, like a hash of the manifest
+// - *common.ParsedDockerURL: a parsed docker URL
+// - error: an error if one occurred
+func (lb *FileBackend) GetImageInfo(dockerURL string) ([]string, string, *common.ParsedDockerURL, error) {
 	// a missing Docker URL could mean that the file only contains one
 	// image so it's okay for dockerURL to be blank
 	var parsedDockerURL *common.ParsedDockerURL
 	if dockerURL != "" {
 		var err error
 		parsedDockerURL, err = common.ParseDockerURL(dockerURL)
 		if err != nil {
-			return nil, nil, fmt.Errorf("image provided couldnot be parsed: %v", err)
+			return nil, "", nil, fmt.Errorf("image provided couldnot be parsed: %v", err)
 		}
 	}
 
@@ -70,25 +76,25 @@ func (lb *FileBackend) GetImageInfo(dockerURL string) ([]string, *common.ParsedD
 	name := strings.Split(filepath.Base(lb.file.Name()), ".")[0]
 	appImageID, ancestry, parsedDockerURL, err := getImageID(lb.file, parsedDockerURL, name, lb.debug)
 	if err != nil {
-		return nil, nil, err
+		return nil, "", nil, err
 	}
 
 	if len(ancestry) == 0 {
 		ancestry, err = getAncestry(lb.file, appImageID, lb.debug)
 		if err != nil {
-			return nil, nil, fmt.Errorf("error getting ancestry: %v", err)
+			return nil, "", nil, fmt.Errorf("error getting ancestry: %v", err)
 		}
 	} else {
 		// for oci the first image is the config
 		ancestry = append([]string{appImageID}, ancestry...)
 	}
 
-	return ancestry, parsedDockerURL, nil
+	return ancestry, appImageID, parsedDockerURL, nil
 }
 
-func (lb *FileBackend) BuildACI(layerIDs []string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
+func (lb *FileBackend) BuildACI(layerIDs []string, manhash string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
 	if strings.Contains(layerIDs[0], ":") {
-		return lb.BuildACIV22(layerIDs, dockerURL, outputDir, tmpBaseDir, compression)
+		return lb.BuildACIV22(layerIDs, manhash, dockerURL, outputDir, tmpBaseDir, compression)
 	}
 	var aciLayerPaths []string
 	var aciManifests []*schema.ImageManifest
@@ -125,7 +131,7 @@ func (lb *FileBackend) BuildACI(layerIDs []string, dockerURL *common.ParsedDocke
 		defer layerFile.Close()
 
 		lb.debug.Println("Generating layer ACI...")
-		aciPath, manifest, err := internal.GenerateACI(i, layerData, dockerURL, outputDir, layerFile, curPwl, compression, lb.debug)
+		aciPath, manifest, err := internal.GenerateACI(i, manhash, layerData, dockerURL, outputDir, layerFile, curPwl, compression, lb.debug)
 		if err != nil {
 			return nil, nil, fmt.Errorf("error generating ACI: %v", err)
 		}
@@ -138,7 +144,7 @@ func (lb *FileBackend) BuildACI(layerIDs []string, dockerURL *common.ParsedDocke
 	return aciLayerPaths, aciManifests, nil
 }
 
-func (lb *FileBackend) BuildACIV22(layerIDs []string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
+func (lb *FileBackend) BuildACIV22(layerIDs []string, manhash string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
 	if len(layerIDs) < 2 {
 		return nil, nil, fmt.Errorf("insufficient layers for oci image")
 	}
@@ -179,7 +185,7 @@ func (lb *FileBackend) BuildACIV22(layerIDs []string, dockerURL *common.ParsedDo
 		if i != 0 {
 			aciPath, manifest, err = internal.GenerateACI22LowerLayer(dockerURL, parts[1], outputDir, layerFile, curPwl, compression)
 		} else {
-			aciPath, manifest, err = internal.GenerateACI22TopLayer(dockerURL, &imageConfig, parts[1], outputDir, layerFile, curPwl, compression, aciManifests, lb.debug)
+			aciPath, manifest, err = internal.GenerateACI22TopLayer(dockerURL, manhash, &imageConfig, parts[1], outputDir, layerFile, curPwl, compression, aciManifests, lb.debug)
 		}
 		if err != nil {
 			return nil, nil, fmt.Errorf("error generating ACI: %v", err)

lib/internal/backend/repository/repository.go

@@ -87,10 +87,16 @@ func NewRepositoryBackend(username string, password string, insecure common.Inse
 	}
 }
 
-func (rb *RepositoryBackend) GetImageInfo(url string) ([]string, *common.ParsedDockerURL, error) {
+// GetImageInfo, given the url for a docker image, will return the
+// following:
+// - []string: an ordered list of all layer hashes
+// - string: a unique identifier for this image, like a hash of the manifest
+// - *common.ParsedDockerURL: a parsed docker URL
+// - error: an error if one occurred
+func (rb *RepositoryBackend) GetImageInfo(url string) ([]string, string, *common.ParsedDockerURL, error) {
 	dockerURL, err := common.ParseDockerURL(url)
 	if err != nil {
-		return nil, nil, err
+		return nil, "", nil, err
 	}
 
 	var supportsV2, supportsV1, ok bool
@@ -100,42 +106,42 @@ func (rb *RepositoryBackend) GetImageInfo(url string) ([]string, *common.ParsedD
 		var err error
 		URLSchema, supportsV2, err = rb.supportsRegistry(dockerURL.IndexURL, registryV2)
 		if err != nil {
-			return nil, nil, err
+			return nil, "", nil, err
 		}
 		rb.schema = URLSchema + "://"
 		rb.hostsV2Support[dockerURL.IndexURL] = supportsV2
 	}
 
 	// try v2
 	if supportsV2 {
-		layers, dockerURL, err := rb.getImageInfoV2(dockerURL)
+		layers, manhash, dockerURL, err := rb.getImageInfoV2(dockerURL)
 		if !isErrHTTP404(err) {
-			return layers, dockerURL, err
+			return layers, manhash, dockerURL, err
 		}
 		// fallback on 404 failure
 		rb.hostsV1fallback = true
 	}
 
 	URLSchema, supportsV1, err = rb.supportsRegistry(dockerURL.IndexURL, registryV1)
 	if err != nil {
-		return nil, nil, err
+		return nil, "", nil, err
 	}
 	if !supportsV1 && rb.hostsV1fallback {
-		return nil, nil, fmt.Errorf("attempted fallback to API v1 but not supported")
+		return nil, "", nil, fmt.Errorf("attempted fallback to API v1 but not supported")
 	}
 	if !supportsV1 && !supportsV2 {
-		return nil, nil, fmt.Errorf("registry doesn't support API v2 nor v1")
+		return nil, "", nil, fmt.Errorf("registry doesn't support API v2 nor v1")
 	}
 	rb.schema = URLSchema + "://"
 	// try v1, hard fail on failure
 	return rb.getImageInfoV1(dockerURL)
 }
 
-func (rb *RepositoryBackend) BuildACI(layerIDs []string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
+func (rb *RepositoryBackend) BuildACI(layerIDs []string, manhash string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
 	if rb.hostsV1fallback || !rb.hostsV2Support[dockerURL.IndexURL] {
-		return rb.buildACIV1(layerIDs, dockerURL, outputDir, tmpBaseDir, compression)
+		return rb.buildACIV1(layerIDs, manhash, dockerURL, outputDir, tmpBaseDir, compression)
 	} else {
-		return rb.buildACIV2(layerIDs, dockerURL, outputDir, tmpBaseDir, compression)
+		return rb.buildACIV2(layerIDs, manhash, dockerURL, outputDir, tmpBaseDir, compression)
 	}
 }
 

lib/internal/backend/repository/repository1.go

@@ -40,29 +40,29 @@ type RepoData struct {
 	Cookie    []string
 }
 
-func (rb *RepositoryBackend) getImageInfoV1(dockerURL *common.ParsedDockerURL) ([]string, *common.ParsedDockerURL, error) {
+func (rb *RepositoryBackend) getImageInfoV1(dockerURL *common.ParsedDockerURL) ([]string, string, *common.ParsedDockerURL, error) {
 	repoData, err := rb.getRepoDataV1(dockerURL.IndexURL, dockerURL.ImageName)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error getting repository data: %v", err)
+		return nil, "", nil, fmt.Errorf("error getting repository data: %v", err)
 	}
 
 	// TODO(iaguis) check more endpoints
 	appImageID, err := rb.getImageIDFromTagV1(repoData.Endpoints[0], dockerURL.ImageName, dockerURL.Tag, repoData)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error getting ImageID from tag %s: %v", dockerURL.Tag, err)
+		return nil, "", nil, fmt.Errorf("error getting ImageID from tag %s: %v", dockerURL.Tag, err)
 	}
 
 	ancestry, err := rb.getAncestryV1(appImageID, repoData.Endpoints[0], repoData)
 	if err != nil {
-		return nil, nil, err
+		return nil, "", nil, err
 	}
 
 	rb.repoData = repoData
 
-	return ancestry, dockerURL, nil
+	return ancestry, appImageID, dockerURL, nil
 }
 
-func (rb *RepositoryBackend) buildACIV1(layerIDs []string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
+func (rb *RepositoryBackend) buildACIV1(layerIDs []string, manhash string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
 	layerFiles := make([]*os.File, len(layerIDs))
 	layerDatas := make([]types.DockerImageData, len(layerIDs))
 
@@ -121,7 +121,7 @@ func (rb *RepositoryBackend) buildACIV1(layerIDs []string, dockerURL *common.Par
 
 	for i := len(layerIDs) - 1; i >= 0; i-- {
 		rb.debug.Println("Generating layer ACI...")
-		aciPath, manifest, err := internal.GenerateACI(i, layerDatas[i], dockerURL, outputDir, layerFiles[i], curPwl, compression, rb.debug)
+		aciPath, manifest, err := internal.GenerateACI(i, manhash, layerDatas[i], dockerURL, outputDir, layerFiles[i], curPwl, compression, rb.debug)
 		if err != nil {
 			return nil, nil, fmt.Errorf("error generating ACI: %v", err)
 		}