diff --git a/lib/common/common.go b/lib/common/common.go index 8e49b049..13ca559f 100644 --- a/lib/common/common.go +++ b/lib/common/common.go @@ -21,6 +21,8 @@ import ( "github.com/appc/docker2aci/lib/internal/docker" "github.com/docker/distribution/reference" + + spec "github.com/opencontainers/image-spec/specs-go/v1" ) type Compression int @@ -113,3 +115,145 @@ func ValidateLayerId(id string) error { } return nil } + +/* + * Media Type Selectors Section + */ + +const ( + MediaTypeDockerV21Manifest = "application/vnd.docker.distribution.manifest.v1+json" + MediaTypeDockerV21SignedManifest = "application/vnd.docker.distribution.manifest.v1+prettyjws" + MediaTypeDockerV21ManifestLayer = "application/vnd.docker.container.image.rootfs.diff+x-gtar" + + MediaTypeDockerV22Manifest = "application/vnd.docker.distribution.manifest.v2+json" + MediaTypeDockerV22ManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" + MediaTypeDockerV22Config = "application/vnd.docker.container.image.v1+json" + MediaTypeDockerV22RootFS = "application/vnd.docker.image.rootfs.diff.tar.gzip" + + MediaTypeOCIV1Manifest = spec.MediaTypeImageManifest + MediaTypeOCIV1ManifestList = spec.MediaTypeImageManifestList + MediaTypeOCIV1Config = spec.MediaTypeImageConfig + MediaTypeOCIV1Layer = spec.MediaTypeImageLayer +) + +// MediaTypeOption represents the media types for a given docker image (or oci) +// spec. +type MediaTypeOption int + +const ( + MediaTypeOptionDockerV21 = iota + MediaTypeOptionDockerV22 + MediaTypeOptionOCIV1Pre +) + +// MediaTypeSet represents a set of media types which docker2aci is to use when +// fetchimg images. As an example if a MediaTypeSet is equal to +// {MediaTypeOptionDockerV22, MediaTypeOptionOCIV1Pre}, then when an image pull +// is made V2.1 images will not be fetched. This doesn't apply to V1 pulls. As +// an edge case if a MedaTypeSet is nil or empty, that means that _every_ type +// of media type is enabled. This type is intended to be a set, and putting +// duplicates in this set is generally unadvised. +type MediaTypeSet []MediaTypeOption + +func (m MediaTypeSet) ManifestMediaTypes() []string { + if len(m) == 0 { + return []string{ + MediaTypeDockerV21Manifest, + MediaTypeDockerV22Manifest, + MediaTypeOCIV1Manifest, + } + } + ret := []string{} + for _, option := range m { + switch option { + case MediaTypeOptionDockerV21: + ret = append(ret, MediaTypeDockerV21Manifest) + case MediaTypeOptionDockerV22: + ret = append(ret, MediaTypeDockerV22Manifest) + case MediaTypeOptionOCIV1Pre: + ret = append(ret, MediaTypeOCIV1Manifest) + } + } + return ret +} + +func (m MediaTypeSet) ConfigMediaTypes() []string { + if len(m) == 0 { + return []string{ + MediaTypeDockerV22Config, + MediaTypeOCIV1Config, + } + } + ret := []string{} + for _, option := range m { + switch option { + case MediaTypeOptionDockerV21: + case MediaTypeOptionDockerV22: + ret = append(ret, MediaTypeDockerV22Config) + case MediaTypeOptionOCIV1Pre: + ret = append(ret, MediaTypeOCIV1Config) + } + } + return ret +} + +func (m MediaTypeSet) LayerMediaTypes() []string { + if len(m) == 0 { + return []string{ + MediaTypeDockerV22RootFS, + MediaTypeOCIV1Layer, + } + } + ret := []string{} + for _, option := range m { + switch option { + case MediaTypeOptionDockerV21: + case MediaTypeOptionDockerV22: + ret = append(ret, MediaTypeDockerV22RootFS) + case MediaTypeOptionOCIV1Pre: + ret = append(ret, MediaTypeOCIV1Layer) + } + } + return ret +} + +// RegistryOption represents a type of a registry, based on the version of the +// docker http API. +type RegistryOption int + +const ( + RegistryOptionV1 = iota + RegistryOptionV2 +) + +// RegistryOptionSet represents a set of registry types which docker2aci is to +// use when fetching images. As an example if a RegistryOptionSet is equal to +// {RegistryOptionV2}, then v1 pulls are disabled. As an edge case if a +// RegistryOptionSet is nil or empty, that means that _every_ type of registry +// is enabled. This type is intended to be a set, and putting duplicates in this +// set is generally unadvised. +type RegistryOptionSet []RegistryOption + +func (r RegistryOptionSet) AllowsV1() bool { + if len(r) == 0 { + return true + } + for _, o := range r { + if o == RegistryOptionV1 { + return true + } + } + return false +} + +func (r RegistryOptionSet) AllowsV2() bool { + if len(r) == 0 { + return true + } + for _, o := range r { + if o == RegistryOptionV2 { + return true + } + } + return false +} diff --git a/lib/common/common_test.go b/lib/common/common_test.go new file mode 100644 index 00000000..c73e101f --- /dev/null +++ b/lib/common/common_test.go @@ -0,0 +1,122 @@ +// Copyright 2017 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" +) + +func TestMediaTypeSet(t *testing.T) { + tests := []struct { + ms MediaTypeSet + expectedManifestTypes []string + expectedConfigTypes []string + expectedLayerTypes []string + }{ + { + MediaTypeSet{MediaTypeOptionDockerV21}, + []string{MediaTypeDockerV21Manifest}, + []string{}, + []string{}, + }, + { + MediaTypeSet{MediaTypeOptionDockerV22}, + []string{MediaTypeDockerV22Manifest}, + []string{MediaTypeDockerV22Config}, + []string{MediaTypeDockerV22RootFS}, + }, + { + MediaTypeSet{MediaTypeOptionOCIV1Pre}, + []string{MediaTypeOCIV1Manifest}, + []string{MediaTypeOCIV1Config}, + []string{MediaTypeOCIV1Layer}, + }, + { + MediaTypeSet{}, + []string{MediaTypeDockerV21Manifest, MediaTypeDockerV22Manifest, MediaTypeOCIV1Manifest}, + []string{MediaTypeDockerV22Config, MediaTypeOCIV1Config}, + []string{MediaTypeDockerV22RootFS, MediaTypeOCIV1Layer}, + }, + { + MediaTypeSet{MediaTypeOptionDockerV21, MediaTypeOptionDockerV22, MediaTypeOptionOCIV1Pre}, + []string{MediaTypeDockerV21Manifest, MediaTypeDockerV22Manifest, MediaTypeOCIV1Manifest}, + []string{MediaTypeDockerV22Config, MediaTypeOCIV1Config}, + []string{MediaTypeDockerV22RootFS, MediaTypeOCIV1Layer}, + }, + { + MediaTypeSet{MediaTypeOptionDockerV21, MediaTypeOptionOCIV1Pre}, + []string{MediaTypeDockerV21Manifest, MediaTypeOCIV1Manifest}, + []string{MediaTypeOCIV1Config}, + []string{MediaTypeOCIV1Layer}, + }, + } + + for _, test := range tests { + if !isEqual(test.expectedManifestTypes, test.ms.ManifestMediaTypes()) { + t.Errorf("expected manifest media types didn't match what was returned:\n%v\n%v", test.expectedManifestTypes, test.ms.ManifestMediaTypes()) + } + if !isEqual(test.expectedConfigTypes, test.ms.ConfigMediaTypes()) { + t.Errorf("expected config media types didn't match what was returned:\n%v\n%v", test.expectedConfigTypes, test.ms.ConfigMediaTypes()) + } + if !isEqual(test.expectedLayerTypes, test.ms.LayerMediaTypes()) { + t.Errorf("expected layer media types didn't match what was returned:\n%v\n%v", test.expectedLayerTypes, test.ms.LayerMediaTypes()) + } + } +} + +func TestRegistryOptionSet(t *testing.T) { + tests := []struct { + rs RegistryOptionSet + allowsV1 bool + allowsV2 bool + }{ + { + RegistryOptionSet{RegistryOptionV1}, true, false, + }, + { + RegistryOptionSet{RegistryOptionV2}, false, true, + }, + { + RegistryOptionSet{RegistryOptionV1, RegistryOptionV2}, true, true, + }, + { + RegistryOptionSet{}, true, true, + }, + } + for _, test := range tests { + if test.allowsV1 != test.rs.AllowsV1() { + t.Errorf("doesn't allow V1 when it should") + } + if test.allowsV2 != test.rs.AllowsV2() { + t.Errorf("doesn't allow V1 when it should") + } + } +} + +func isEqual(val1, val2 []string) bool { + if len(val1) != len(val2) { + return false + } +loop1: + for _, thing1 := range val1 { + for _, thing2 := range val2 { + if thing1 == thing2 { + continue loop1 + } + } + return false + } + return true +} diff --git a/lib/docker2aci.go b/lib/docker2aci.go index 685fcc2b..b888e649 100644 --- a/lib/docker2aci.go +++ b/lib/docker2aci.go @@ -66,9 +66,11 @@ func (c *CommonConfig) initLogger() { // converting Docker images. type RemoteConfig struct { CommonConfig - Username string // username to use if the image to convert needs authentication - Password string // password to use if the image to convert needs authentication - Insecure common.InsecureConfig // Insecure options + Username string // username to use if the image to convert needs authentication + Password string // password to use if the image to convert needs authentication + Insecure common.InsecureConfig // Insecure options + MediaTypes common.MediaTypeSet + RegistryOptions common.RegistryOptionSet } // FileConfig represents the saved file specific configuration for converting @@ -95,6 +97,8 @@ func ConvertRemoteRepo(dockerURL string, config RemoteConfig) ([]string, error) config.Password, config.Insecure, config.Debug, + config.MediaTypes, + config.RegistryOptions, ), dockerURL: dockerURL, config: config.CommonConfig, diff --git a/lib/internal/backend/repository/repository.go b/lib/internal/backend/repository/repository.go index fa460430..97315fd3 100644 --- a/lib/internal/backend/repository/repository.go +++ b/lib/internal/backend/repository/repository.go @@ -67,11 +67,13 @@ type RepositoryBackend struct { imageV2Manifests map[common.ParsedDockerURL]*typesV2.ImageManifest imageConfigs map[common.ParsedDockerURL]*typesV2.ImageConfig layersIndex map[string]int + mediaTypes common.MediaTypeSet + registryOptions common.RegistryOptionSet debug log.Logger } -func NewRepositoryBackend(username string, password string, insecure common.InsecureConfig, debug log.Logger) *RepositoryBackend { +func NewRepositoryBackend(username, password string, insecure common.InsecureConfig, debug log.Logger, mediaTypes common.MediaTypeSet, registryOptions common.RegistryOptionSet) *RepositoryBackend { return &RepositoryBackend{ username: username, password: password, @@ -83,6 +85,8 @@ func NewRepositoryBackend(username string, password string, insecure common.Inse imageV2Manifests: make(map[common.ParsedDockerURL]*typesV2.ImageManifest), imageConfigs: make(map[common.ParsedDockerURL]*typesV2.ImageConfig), layersIndex: make(map[string]int), + mediaTypes: mediaTypes, + registryOptions: registryOptions, debug: debug, } } @@ -113,13 +117,21 @@ func (rb *RepositoryBackend) GetImageInfo(url string) ([]string, string, *common } // try v2 - if supportsV2 { + if supportsV2 && rb.registryOptions.AllowsV2() { layers, manhash, dockerURL, err := rb.getImageInfoV2(dockerURL) if !isErrHTTP404(err) { return layers, manhash, dockerURL, err } // fallback on 404 failure rb.hostsV1fallback = true + // unless we can't fallback + if !rb.registryOptions.AllowsV1() { + return nil, "", nil, err + } + } + + if !rb.registryOptions.AllowsV1() { + return nil, "", nil, fmt.Errorf("no remaining enabled registry options") } URLSchema, supportsV1, err = rb.supportsRegistry(dockerURL.IndexURL, registryV1) diff --git a/lib/internal/backend/repository/repository2.go b/lib/internal/backend/repository/repository2.go index 41ac3c43..1e03791f 100644 --- a/lib/internal/backend/repository/repository2.go +++ b/lib/internal/backend/repository/repository2.go @@ -295,13 +295,7 @@ func (rb *RepositoryBackend) getManifestV2(dockerURL *common.ParsedDockerURL) ([ rb.setBasicAuth(req) - accepting := []string{ - typesV2.MediaTypeOCIManifest, - typesV2.MediaTypeDockerV22Manifest, - typesV2.MediaTypeDockerV21Manifest, - } - - res, err := rb.makeRequest(req, dockerURL.ImageName, accepting) + res, err := rb.makeRequest(req, dockerURL.ImageName, rb.mediaTypes.ManifestMediaTypes()) if err != nil { return nil, "", err } @@ -312,9 +306,9 @@ func (rb *RepositoryBackend) getManifestV2(dockerURL *common.ParsedDockerURL) ([ } switch res.Header.Get("content-type") { - case typesV2.MediaTypeDockerV22Manifest, typesV2.MediaTypeOCIManifest: + case common.MediaTypeDockerV22Manifest, common.MediaTypeOCIV1Manifest: return rb.getManifestV22(dockerURL, res) - case typesV2.MediaTypeDockerV21Manifest: + case common.MediaTypeDockerV21Manifest: return rb.getManifestV21(dockerURL, res) } return rb.getManifestV21(dockerURL, res) @@ -405,12 +399,7 @@ func (rb *RepositoryBackend) getConfigV22(dockerURL *common.ParsedDockerURL, con rb.setBasicAuth(req) - accepting := []string{ - typesV2.MediaTypeOCIConfig, - typesV2.MediaTypeDockerV22Config, - } - - res, err := rb.makeRequest(req, dockerURL.ImageName, accepting) + res, err := rb.makeRequest(req, dockerURL.ImageName, rb.mediaTypes.ConfigMediaTypes()) if err != nil { return err } @@ -491,12 +480,7 @@ func (rb *RepositoryBackend) getLayerV2(layerID string, dockerURL *common.Parsed rb.setBasicAuth(req) - accepting := []string{ - typesV2.MediaTypeDockerV22RootFS, - typesV2.MediaTypeOCILayer, - } - - res, err = rb.makeRequest(req, dockerURL.ImageName, accepting) + res, err = rb.makeRequest(req, dockerURL.ImageName, rb.mediaTypes.LayerMediaTypes()) if err != nil { return nil, nil, err } @@ -516,7 +500,7 @@ func (rb *RepositoryBackend) getLayerV2(layerID string, dockerURL *common.Parsed } res.Body.Close() res = nil - res, err = rb.makeRequest(req, dockerURL.ImageName, accepting) + res, err = rb.makeRequest(req, dockerURL.ImageName, rb.mediaTypes.LayerMediaTypes()) if err != nil { return nil, nil, err } diff --git a/lib/internal/typesV2/docker_types.go b/lib/internal/typesV2/docker_types.go index f8305f75..8d7f6fb1 100644 --- a/lib/internal/typesV2/docker_types.go +++ b/lib/internal/typesV2/docker_types.go @@ -18,23 +18,7 @@ import ( "encoding/json" "errors" - spec "github.com/opencontainers/image-spec/specs-go/v1" -) - -const ( - MediaTypeDockerV21Manifest = "application/vnd.docker.distribution.manifest.v1+json" - MediaTypeDockerV21SignedManifest = "application/vnd.docker.distribution.manifest.v1+prettyjws" - MediaTypeDockerV21ManifestLayer = "application/vnd.docker.container.image.rootfs.diff+x-gtar" - - MediaTypeDockerV22Manifest = "application/vnd.docker.distribution.manifest.v2+json" - MediaTypeDockerV22ManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" - MediaTypeDockerV22Config = "application/vnd.docker.container.image.v1+json" - MediaTypeDockerV22RootFS = "application/vnd.docker.image.rootfs.diff.tar.gzip" - - MediaTypeOCIManifest = spec.MediaTypeImageManifest - MediaTypeOCIManifestList = spec.MediaTypeImageManifestList - MediaTypeOCIConfig = spec.MediaTypeImageConfig - MediaTypeOCILayer = spec.MediaTypeImageLayer + "github.com/appc/docker2aci/lib/common" ) var ( @@ -74,7 +58,7 @@ func (im *ImageManifest) PrettyString() string { } func (im *ImageManifest) Validate() error { - if im.MediaType != MediaTypeDockerV22Manifest && im.MediaType != MediaTypeOCIManifest { + if im.MediaType != common.MediaTypeDockerV22Manifest && im.MediaType != common.MediaTypeOCIV1Manifest { return ErrIncorrectMediaType } if im.Config == nil { diff --git a/lib/tests/common.go b/lib/tests/common.go index fd1387a8..b5be72a4 100644 --- a/lib/tests/common.go +++ b/lib/tests/common.go @@ -10,6 +10,7 @@ import ( "os" "path" + "github.com/appc/docker2aci/lib/common" "github.com/appc/docker2aci/lib/internal/typesV2" ) @@ -105,9 +106,9 @@ func GenDocker22Manifest(destPath, configHash string, layerHashes []string) erro manifest := &typesV2.ImageManifest{ SchemaVersion: 2, - MediaType: typesV2.MediaTypeDockerV22Manifest, + MediaType: common.MediaTypeDockerV22Manifest, Config: &typesV2.ImageManifestDigest{ - MediaType: typesV2.MediaTypeDockerV22Config, + MediaType: common.MediaTypeDockerV22Config, Size: int(configSize), Digest: "sha256:" + configHash, }, @@ -119,7 +120,7 @@ func GenDocker22Manifest(destPath, configHash string, layerHashes []string) erro } manifest.Layers = append(manifest.Layers, &typesV2.ImageManifestDigest{ - MediaType: typesV2.MediaTypeDockerV22RootFS, + MediaType: common.MediaTypeDockerV22RootFS, Size: int(layerSize), Digest: "sha256:" + h, }) diff --git a/lib/tests/v22_test.go b/lib/tests/v22_test.go index fe670c15..da63537d 100644 --- a/lib/tests/v22_test.go +++ b/lib/tests/v22_test.go @@ -230,7 +230,7 @@ func TestFetchingByTagV22(t *testing.T) { } imgName := "docker2aci/dockerv22test" imgRef := "v0.1.0" - server := RunDockerRegistry(t, tmpDir, imgName, imgRef, typesV2.MediaTypeDockerV22Manifest) + server := RunDockerRegistry(t, tmpDir, imgName, imgRef, d2acommon.MediaTypeDockerV22Manifest) defer server.Close() bareServerURL := strings.TrimPrefix(server.URL, "http://") @@ -347,7 +347,7 @@ func TestFetchingByDigestV22(t *testing.T) { } imgName := "docker2aci/dockerv22test" imgRef := "sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2" - server := RunDockerRegistry(t, tmpDir, imgName, imgRef, typesV2.MediaTypeDockerV22Manifest) + server := RunDockerRegistry(t, tmpDir, imgName, imgRef, d2acommon.MediaTypeDockerV22Manifest) defer server.Close() localUrl := path.Join(strings.TrimPrefix(server.URL, "http://"), imgName) + "@" + imgRef @@ -396,7 +396,7 @@ func TestFetchingMultipleLayersV22(t *testing.T) { } imgName := "docker2aci/dockerv22test" imgRef := "v0.1.0" - server := RunDockerRegistry(t, tmpDir, imgName, imgRef, typesV2.MediaTypeDockerV22Manifest) + server := RunDockerRegistry(t, tmpDir, imgName, imgRef, d2acommon.MediaTypeDockerV22Manifest) defer server.Close() localUrl := path.Join(strings.TrimPrefix(server.URL, "http://"), imgName) + ":" + imgRef diff --git a/tests/test.sh b/tests/test.sh index 27567542..52526210 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -21,6 +21,7 @@ go vet ./pkg/... go vet ./lib/... go test -v ${REPO_PATH}/lib/tests go test -v ${REPO_PATH}/lib/internal +go test -v ${REPO_PATH}/lib/common DOCKER2ACI=../bin/docker2aci PREFIX=docker2aci-tests