backend/repository: pull layers in parallel

To reduce the time it takes to fetch docker images, this commit modifies the backends to pull each image layer concurrently. As a result, github.com/coreos/pkg/progressutil is used in place of ioprogress to draw the progress bars.
appc · May 25, 2016 · 47f2cb9 · 47f2cb9
1 parent 572a305
commit 47f2cb9
Show file tree

Hide file tree

Showing 6 changed files with 204 additions and 133 deletions.
diff --git a/lib/docker2aci.go b/lib/docker2aci.go
@@ -124,31 +124,25 @@ func convertReal(backend internal.Docker2ACIBackend, dockerURL string, squash bo
 
 	conversionStore := newConversionStore()
 
-	var images acirenderer.Images
-	var aciLayerPaths []string
-	var curPwl []string
-	for i := len(ancestry) - 1; i >= 0; i-- {
-		layerID := ancestry[i]
-
-		// only compress individual layers if we're not squashing
-		layerCompression := compression
-		if squash {
-			layerCompression = common.NoCompression
-		}
+	// only compress individual layers if we're not squashing
+	layerCompression := compression
+	if squash {
+		layerCompression = common.NoCompression
+	}
 
-		aciPath, manifest, err := backend.BuildACI(i, layerID, parsedDockerURL, layersOutputDir, tmpDir, curPwl, layerCompression)
-		if err != nil {
-			return nil, fmt.Errorf("error building layer: %v", err)
-		}
+	aciLayerPaths, aciManifests, err := backend.BuildACI(ancestry, parsedDockerURL, layersOutputDir, tmpDir, layerCompression)
+	if err != nil {
+		return nil, err
+	}
 
-		key, err := conversionStore.WriteACI(aciPath)
+	var images acirenderer.Images
+	for i, aciLayerPath := range aciLayerPaths {
+		key, err := conversionStore.WriteACI(aciLayerPath)
 		if err != nil {
 			return nil, fmt.Errorf("error inserting in the conversion store: %v", err)
 		}
 
-		images = append(images, acirenderer.Image{Im: manifest, Key: key, Level: uint16(i)})
-		aciLayerPaths = append(aciLayerPaths, aciPath)
-		curPwl = manifest.PathWhitelist
+		images = append(images, acirenderer.Image{Im: aciManifests[i], Key: key, Level: uint16(len(aciLayerPaths) - 1 - i)})
 	}
 
 	// acirenderer expects images in order from upper to base layer

diff --git a/lib/internal/backend/file/file.go b/lib/internal/backend/file/file.go
@@ -68,39 +68,48 @@ func (lb *FileBackend) GetImageInfo(dockerURL string) ([]string, *types.ParsedDo
 	return ancestry, parsedDockerURL, nil
 }
 
-func (lb *FileBackend) BuildACI(layerNumber int, layerID string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, curPwl []string, compression common.Compression) (string, *schema.ImageManifest, error) {
-	tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-")
-	if err != nil {
-		return "", nil, fmt.Errorf("error creating dir: %v", err)
-	}
-	defer os.RemoveAll(tmpDir)
+func (lb *FileBackend) BuildACI(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
+	var aciLayerPaths []string
+	var aciManifests []*schema.ImageManifest
+	var curPwl []string
+	for i := len(layerIDs) - 1; i >= 0; i-- {
+		tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-")
+		if err != nil {
+			return nil, nil, fmt.Errorf("error creating dir: %v", err)
+		}
+		defer os.RemoveAll(tmpDir)
 
-	j, err := getJson(lb.file, layerID)
-	if err != nil {
-		return "", nil, fmt.Errorf("error getting image json: %v", err)
-	}
+		j, err := getJson(lb.file, layerIDs[i])
+		if err != nil {
+			return nil, nil, fmt.Errorf("error getting image json: %v", err)
+		}
 
-	layerData := types.DockerImageData{}
-	if err := json.Unmarshal(j, &layerData); err != nil {
-		return "", nil, fmt.Errorf("error unmarshaling layer data: %v", err)
-	}
+		layerData := types.DockerImageData{}
+		if err := json.Unmarshal(j, &layerData); err != nil {
+			return nil, nil, fmt.Errorf("error unmarshaling layer data: %v", err)
+		}
 
-	tmpLayerPath := path.Join(tmpDir, layerID)
-	tmpLayerPath += ".tar"
+		tmpLayerPath := path.Join(tmpDir, layerIDs[i])
+		tmpLayerPath += ".tar"
 
-	layerFile, err := extractEmbeddedLayer(lb.file, layerID, tmpLayerPath)
-	if err != nil {
-		return "", nil, fmt.Errorf("error getting layer from file: %v", err)
-	}
-	defer layerFile.Close()
+		layerFile, err := extractEmbeddedLayer(lb.file, layerIDs[i], tmpLayerPath)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error getting layer from file: %v", err)
+		}
+		defer layerFile.Close()
 
-	log.Debug("Generating layer ACI...")
-	aciPath, manifest, err := internal.GenerateACI(layerNumber, layerData, dockerURL, outputDir, layerFile, curPwl, compression)
-	if err != nil {
-		return "", nil, fmt.Errorf("error generating ACI: %v", err)
+		log.Debug("Generating layer ACI...")
+		aciPath, manifest, err := internal.GenerateACI(i, layerData, dockerURL, outputDir, layerFile, curPwl, compression)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error generating ACI: %v", err)
+		}
+
+		aciLayerPaths = append(aciLayerPaths, aciPath)
+		aciManifests = append(aciManifests, manifest)
+		curPwl = manifest.PathWhitelist
 	}
 
-	return aciPath, manifest, nil
+	return aciLayerPaths, aciManifests, nil
 }
 
 func getImageID(file *os.File, dockerURL *types.ParsedDockerURL) (string, *types.ParsedDockerURL, error) {

diff --git a/lib/internal/backend/repository/repository.go b/lib/internal/backend/repository/repository.go
@@ -93,11 +93,11 @@ func (rb *RepositoryBackend) GetImageInfo(url string) ([]string, *types.ParsedDo
 	}
 }
 
-func (rb *RepositoryBackend) BuildACI(layerNumber int, layerID string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, curPwl []string, compression common.Compression) (string, *schema.ImageManifest, error) {
+func (rb *RepositoryBackend) BuildACI(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
 	if rb.hostsV2Support[dockerURL.IndexURL] {
-		return rb.buildACIV2(layerNumber, layerID, dockerURL, outputDir, tmpBaseDir, curPwl, compression)
+		return rb.buildACIV2(layerIDs, dockerURL, outputDir, tmpBaseDir, compression)
 	} else {
-		return rb.buildACIV1(layerNumber, layerID, dockerURL, outputDir, tmpBaseDir, curPwl, compression)
+		return rb.buildACIV1(layerIDs, dockerURL, outputDir, tmpBaseDir, compression)
 	}
 }
 

diff --git a/lib/internal/backend/repository/repository1.go b/lib/internal/backend/repository/repository1.go
@@ -62,36 +62,73 @@ func (rb *RepositoryBackend) getImageInfoV1(dockerURL *types.ParsedDockerURL) ([
 	return ancestry, dockerURL, nil
 }
 
-func (rb *RepositoryBackend) buildACIV1(layerNumber int, layerID string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, curPwl []string, compression common.Compression) (string, *schema.ImageManifest, error) {
-	tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-")
-	if err != nil {
-		return "", nil, fmt.Errorf("error creating dir: %v", err)
-	}
-	defer os.RemoveAll(tmpDir)
+func (rb *RepositoryBackend) buildACIV1(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
+	layerFiles := make([]*os.File, len(layerIDs))
+	layerDatas := make([]types.DockerImageData, len(layerIDs))
 
-	j, size, err := rb.getJsonV1(layerID, rb.repoData.Endpoints[0], rb.repoData)
+	tmpParentDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-")
 	if err != nil {
-		return "", nil, fmt.Errorf("error getting image json: %v", err)
+		return nil, nil, err
 	}
+	defer os.RemoveAll(tmpParentDir)
+
+	var doneChannels []chan error
+	for i, layerID := range layerIDs {
+		doneChan := make(chan error)
+		doneChannels = append(doneChannels, doneChan)
+		// https://github.com/golang/go/wiki/CommonMistakes
+		i := i // golang--
+		layerID := layerID
+		go func() {
+			tmpDir, err := ioutil.TempDir(tmpParentDir, "")
+			if err != nil {
+				doneChan <- fmt.Errorf("error creating dir: %v", err)
+				return
+			}
 
-	layerData := types.DockerImageData{}
-	if err := json.Unmarshal(j, &layerData); err != nil {
-		return "", nil, fmt.Errorf("error unmarshaling layer data: %v", err)
-	}
+			j, size, err := rb.getJsonV1(layerID, rb.repoData.Endpoints[0], rb.repoData)
+			if err != nil {
+				doneChan <- fmt.Errorf("error getting image json: %v", err)
+				return
+			}
 
-	layerFile, err := rb.getLayerV1(layerID, rb.repoData.Endpoints[0], rb.repoData, size, tmpDir)
-	if err != nil {
-		return "", nil, fmt.Errorf("error getting the remote layer: %v", err)
+			layerDatas[i] = types.DockerImageData{}
+			if err := json.Unmarshal(j, &layerDatas[i]); err != nil {
+				doneChan <- fmt.Errorf("error unmarshaling layer data: %v", err)
+				return
+			}
+
+			layerFiles[i], err = rb.getLayerV1(layerID, rb.repoData.Endpoints[0], rb.repoData, size, tmpDir)
+			if err != nil {
+				doneChan <- fmt.Errorf("error getting the remote layer: %v", err)
+				return
+			}
+		}()
+	}
+	for _, doneChan := range doneChannels {
+		err := <-doneChan
+		if err != nil {
+			return nil, nil, err
+		}
 	}
-	defer layerFile.Close()
+	var aciLayerPaths []string
+	var aciManifests []*schema.ImageManifest
+	var curPwl []string
 
-	log.Debug("Generating layer ACI...")
-	aciPath, manifest, err := internal.GenerateACI(layerNumber, layerData, dockerURL, outputDir, layerFile, curPwl, compression)
-	if err != nil {
-		return "", nil, fmt.Errorf("error generating ACI: %v", err)
+	for i := len(layerIDs) - 1; i >= 0; i-- {
+		log.Debug("Generating layer ACI...")
+		aciPath, manifest, err := internal.GenerateACI(i, layerDatas[i], dockerURL, outputDir, layerFiles[i], curPwl, compression)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error generating ACI: %v", err)
+		}
+		aciLayerPaths = append(aciLayerPaths, aciPath)
+		aciManifests = append(aciManifests, manifest)
+		curPwl = manifest.PathWhitelist
+
+		layerFiles[i].Close()
 	}
 
-	return aciPath, manifest, nil
+	return aciLayerPaths, aciManifests, nil
 }
 
 func (rb *RepositoryBackend) getRepoDataV1(indexURL string, remote string) (*RepoData, error) {