From a4e343c2c7859f8fe51856f3f0bc813a5c28a0cd Mon Sep 17 00:00:00 2001 From: Derek Gonyeo Date: Thu, 19 Jan 2017 14:15:40 -0800 Subject: [PATCH] Allow selective disabling of registries and media types This commit tweaks the library's API to allow consumers to selectively disable registry support (v1 or v2) and what mediatypes docker2aci claims to be able to accept during an image pull (v2.1, v2.2, and oci). By default all options are enabled, and this is still the case when using docker2aci via the command line. --- lib/common/common.go | 144 ++++++++++++++++++ lib/common/common_test.go | 122 +++++++++++++++ lib/docker2aci.go | 10 +- lib/internal/backend/repository/repository.go | 16 +- .../backend/repository/repository2.go | 28 +--- lib/internal/typesV2/docker_types.go | 20 +-- lib/tests/common.go | 7 +- lib/tests/v22_test.go | 6 +- tests/test.sh | 1 + 9 files changed, 303 insertions(+), 51 deletions(-) create mode 100644 lib/common/common_test.go diff --git a/lib/common/common.go b/lib/common/common.go index 8e49b049..13ca559f 100644 --- a/lib/common/common.go +++ b/lib/common/common.go @@ -21,6 +21,8 @@ import ( "github.com/appc/docker2aci/lib/internal/docker" "github.com/docker/distribution/reference" + + spec "github.com/opencontainers/image-spec/specs-go/v1" ) type Compression int @@ -113,3 +115,145 @@ func ValidateLayerId(id string) error { } return nil } + +/* + * Media Type Selectors Section + */ + +const ( + MediaTypeDockerV21Manifest = "application/vnd.docker.distribution.manifest.v1+json" + MediaTypeDockerV21SignedManifest = "application/vnd.docker.distribution.manifest.v1+prettyjws" + MediaTypeDockerV21ManifestLayer = "application/vnd.docker.container.image.rootfs.diff+x-gtar" + + MediaTypeDockerV22Manifest = "application/vnd.docker.distribution.manifest.v2+json" + MediaTypeDockerV22ManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" + MediaTypeDockerV22Config = "application/vnd.docker.container.image.v1+json" + MediaTypeDockerV22RootFS = "application/vnd.docker.image.rootfs.diff.tar.gzip" + + MediaTypeOCIV1Manifest = spec.MediaTypeImageManifest + MediaTypeOCIV1ManifestList = spec.MediaTypeImageManifestList + MediaTypeOCIV1Config = spec.MediaTypeImageConfig + MediaTypeOCIV1Layer = spec.MediaTypeImageLayer +) + +// MediaTypeOption represents the media types for a given docker image (or oci) +// spec. +type MediaTypeOption int + +const ( + MediaTypeOptionDockerV21 = iota + MediaTypeOptionDockerV22 + MediaTypeOptionOCIV1Pre +) + +// MediaTypeSet represents a set of media types which docker2aci is to use when +// fetchimg images. As an example if a MediaTypeSet is equal to +// {MediaTypeOptionDockerV22, MediaTypeOptionOCIV1Pre}, then when an image pull +// is made V2.1 images will not be fetched. This doesn't apply to V1 pulls. As +// an edge case if a MedaTypeSet is nil or empty, that means that _every_ type +// of media type is enabled. This type is intended to be a set, and putting +// duplicates in this set is generally unadvised. +type MediaTypeSet []MediaTypeOption + +func (m MediaTypeSet) ManifestMediaTypes() []string { + if len(m) == 0 { + return []string{ + MediaTypeDockerV21Manifest, + MediaTypeDockerV22Manifest, + MediaTypeOCIV1Manifest, + } + } + ret := []string{} + for _, option := range m { + switch option { + case MediaTypeOptionDockerV21: + ret = append(ret, MediaTypeDockerV21Manifest) + case MediaTypeOptionDockerV22: + ret = append(ret, MediaTypeDockerV22Manifest) + case MediaTypeOptionOCIV1Pre: + ret = append(ret, MediaTypeOCIV1Manifest) + } + } + return ret +} + +func (m MediaTypeSet) ConfigMediaTypes() []string { + if len(m) == 0 { + return []string{ + MediaTypeDockerV22Config, + MediaTypeOCIV1Config, + } + } + ret := []string{} + for _, option := range m { + switch option { + case MediaTypeOptionDockerV21: + case MediaTypeOptionDockerV22: + ret = append(ret, MediaTypeDockerV22Config) + case MediaTypeOptionOCIV1Pre: + ret = append(ret, MediaTypeOCIV1Config) + } + } + return ret +} + +func (m MediaTypeSet) LayerMediaTypes() []string { + if len(m) == 0 { + return []string{ + MediaTypeDockerV22RootFS, + MediaTypeOCIV1Layer, + } + } + ret := []string{} + for _, option := range m { + switch option { + case MediaTypeOptionDockerV21: + case MediaTypeOptionDockerV22: + ret = append(ret, MediaTypeDockerV22RootFS) + case MediaTypeOptionOCIV1Pre: + ret = append(ret, MediaTypeOCIV1Layer) + } + } + return ret +} + +// RegistryOption represents a type of a registry, based on the version of the +// docker http API. +type RegistryOption int + +const ( + RegistryOptionV1 = iota + RegistryOptionV2 +) + +// RegistryOptionSet represents a set of registry types which docker2aci is to +// use when fetching images. As an example if a RegistryOptionSet is equal to +// {RegistryOptionV2}, then v1 pulls are disabled. As an edge case if a +// RegistryOptionSet is nil or empty, that means that _every_ type of registry +// is enabled. This type is intended to be a set, and putting duplicates in this +// set is generally unadvised. +type RegistryOptionSet []RegistryOption + +func (r RegistryOptionSet) AllowsV1() bool { + if len(r) == 0 { + return true + } + for _, o := range r { + if o == RegistryOptionV1 { + return true + } + } + return false +} + +func (r RegistryOptionSet) AllowsV2() bool { + if len(r) == 0 { + return true + } + for _, o := range r { + if o == RegistryOptionV2 { + return true + } + } + return false +} diff --git a/lib/common/common_test.go b/lib/common/common_test.go new file mode 100644 index 00000000..c73e101f --- /dev/null +++ b/lib/common/common_test.go @@ -0,0 +1,122 @@ +// Copyright 2017 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" +) + +func TestMediaTypeSet(t *testing.T) { + tests := []struct { + ms MediaTypeSet + expectedManifestTypes []string + expectedConfigTypes []string + expectedLayerTypes []string + }{ + { + MediaTypeSet{MediaTypeOptionDockerV21}, + []string{MediaTypeDockerV21Manifest}, + []string{}, + []string{}, + }, + { + MediaTypeSet{MediaTypeOptionDockerV22}, + []string{MediaTypeDockerV22Manifest}, + []string{MediaTypeDockerV22Config}, + []string{MediaTypeDockerV22RootFS}, + }, + { + MediaTypeSet{MediaTypeOptionOCIV1Pre}, + []string{MediaTypeOCIV1Manifest}, + []string{MediaTypeOCIV1Config}, + []string{MediaTypeOCIV1Layer}, + }, + { + MediaTypeSet{}, + []string{MediaTypeDockerV21Manifest, MediaTypeDockerV22Manifest, MediaTypeOCIV1Manifest}, + []string{MediaTypeDockerV22Config, MediaTypeOCIV1Config}, + []string{MediaTypeDockerV22RootFS, MediaTypeOCIV1Layer}, + }, + { + MediaTypeSet{MediaTypeOptionDockerV21, MediaTypeOptionDockerV22, MediaTypeOptionOCIV1Pre}, + []string{MediaTypeDockerV21Manifest, MediaTypeDockerV22Manifest, MediaTypeOCIV1Manifest}, + []string{MediaTypeDockerV22Config, MediaTypeOCIV1Config}, + []string{MediaTypeDockerV22RootFS, MediaTypeOCIV1Layer}, + }, + { + MediaTypeSet{MediaTypeOptionDockerV21, MediaTypeOptionOCIV1Pre}, + []string{MediaTypeDockerV21Manifest, MediaTypeOCIV1Manifest}, + []string{MediaTypeOCIV1Config}, + []string{MediaTypeOCIV1Layer}, + }, + } + + for _, test := range tests { + if !isEqual(test.expectedManifestTypes, test.ms.ManifestMediaTypes()) { + t.Errorf("expected manifest media types didn't match what was returned:\n%v\n%v", test.expectedManifestTypes, test.ms.ManifestMediaTypes()) + } + if !isEqual(test.expectedConfigTypes, test.ms.ConfigMediaTypes()) { + t.Errorf("expected config media types didn't match what was returned:\n%v\n%v", test.expectedConfigTypes, test.ms.ConfigMediaTypes()) + } + if !isEqual(test.expectedLayerTypes, test.ms.LayerMediaTypes()) { + t.Errorf("expected layer media types didn't match what was returned:\n%v\n%v", test.expectedLayerTypes, test.ms.LayerMediaTypes()) + } + } +} + +func TestRegistryOptionSet(t *testing.T) { + tests := []struct { + rs RegistryOptionSet + allowsV1 bool + allowsV2 bool + }{ + { + RegistryOptionSet{RegistryOptionV1}, true, false, + }, + { + RegistryOptionSet{RegistryOptionV2}, false, true, + }, + { + RegistryOptionSet{RegistryOptionV1, RegistryOptionV2}, true, true, + }, + { + RegistryOptionSet{}, true, true, + }, + } + for _, test := range tests { + if test.allowsV1 != test.rs.AllowsV1() { + t.Errorf("doesn't allow V1 when it should") + } + if test.allowsV2 != test.rs.AllowsV2() { + t.Errorf("doesn't allow V1 when it should") + } + } +} + +func isEqual(val1, val2 []string) bool { + if len(val1) != len(val2) { + return false + } +loop1: + for _, thing1 := range val1 { + for _, thing2 := range val2 { + if thing1 == thing2 { + continue loop1 + } + } + return false + } + return true +} diff --git a/lib/docker2aci.go b/lib/docker2aci.go index 685fcc2b..b888e649 100644 --- a/lib/docker2aci.go +++ b/lib/docker2aci.go @@ -66,9 +66,11 @@ func (c *CommonConfig) initLogger() { // converting Docker images. type RemoteConfig struct { CommonConfig - Username string // username to use if the image to convert needs authentication - Password string // password to use if the image to convert needs authentication - Insecure common.InsecureConfig // Insecure options + Username string // username to use if the image to convert needs authentication + Password string // password to use if the image to convert needs authentication + Insecure common.InsecureConfig // Insecure options + MediaTypes common.MediaTypeSet + RegistryOptions common.RegistryOptionSet } // FileConfig represents the saved file specific configuration for converting @@ -95,6 +97,8 @@ func ConvertRemoteRepo(dockerURL string, config RemoteConfig) ([]string, error) config.Password, config.Insecure, config.Debug, + config.MediaTypes, + config.RegistryOptions, ), dockerURL: dockerURL, config: config.CommonConfig, diff --git a/lib/internal/backend/repository/repository.go b/lib/internal/backend/repository/repository.go index fa460430..97315fd3 100644 --- a/lib/internal/backend/repository/repository.go +++ b/lib/internal/backend/repository/repository.go @@ -67,11 +67,13 @@ type RepositoryBackend struct { imageV2Manifests map[common.ParsedDockerURL]*typesV2.ImageManifest imageConfigs map[common.ParsedDockerURL]*typesV2.ImageConfig layersIndex map[string]int + mediaTypes common.MediaTypeSet + registryOptions common.RegistryOptionSet debug log.Logger } -func NewRepositoryBackend(username string, password string, insecure common.InsecureConfig, debug log.Logger) *RepositoryBackend { +func NewRepositoryBackend(username, password string, insecure common.InsecureConfig, debug log.Logger, mediaTypes common.MediaTypeSet, registryOptions common.RegistryOptionSet) *RepositoryBackend { return &RepositoryBackend{ username: username, password: password, @@ -83,6 +85,8 @@ func NewRepositoryBackend(username string, password string, insecure common.Inse imageV2Manifests: make(map[common.ParsedDockerURL]*typesV2.ImageManifest), imageConfigs: make(map[common.ParsedDockerURL]*typesV2.ImageConfig), layersIndex: make(map[string]int), + mediaTypes: mediaTypes, + registryOptions: registryOptions, debug: debug, } } @@ -113,13 +117,21 @@ func (rb *RepositoryBackend) GetImageInfo(url string) ([]string, string, *common } // try v2 - if supportsV2 { + if supportsV2 && rb.registryOptions.AllowsV2() { layers, manhash, dockerURL, err := rb.getImageInfoV2(dockerURL) if !isErrHTTP404(err) { return layers, manhash, dockerURL, err } // fallback on 404 failure rb.hostsV1fallback = true + // unless we can't fallback + if !rb.registryOptions.AllowsV1() { + return nil, "", nil, err + } + } + + if !rb.registryOptions.AllowsV1() { + return nil, "", nil, fmt.Errorf("no remaining enabled registry options") } URLSchema, supportsV1, err = rb.supportsRegistry(dockerURL.IndexURL, registryV1) diff --git a/lib/internal/backend/repository/repository2.go b/lib/internal/backend/repository/repository2.go index 41ac3c43..1e03791f 100644 --- a/lib/internal/backend/repository/repository2.go +++ b/lib/internal/backend/repository/repository2.go @@ -295,13 +295,7 @@ func (rb *RepositoryBackend) getManifestV2(dockerURL *common.ParsedDockerURL) ([ rb.setBasicAuth(req) - accepting := []string{ - typesV2.MediaTypeOCIManifest, - typesV2.MediaTypeDockerV22Manifest, - typesV2.MediaTypeDockerV21Manifest, - } - - res, err := rb.makeRequest(req, dockerURL.ImageName, accepting) + res, err := rb.makeRequest(req, dockerURL.ImageName, rb.mediaTypes.ManifestMediaTypes()) if err != nil { return nil, "", err } @@ -312,9 +306,9 @@ func (rb *RepositoryBackend) getManifestV2(dockerURL *common.ParsedDockerURL) ([ } switch res.Header.Get("content-type") { - case typesV2.MediaTypeDockerV22Manifest, typesV2.MediaTypeOCIManifest: + case common.MediaTypeDockerV22Manifest, common.MediaTypeOCIV1Manifest: return rb.getManifestV22(dockerURL, res) - case typesV2.MediaTypeDockerV21Manifest: + case common.MediaTypeDockerV21Manifest: return rb.getManifestV21(dockerURL, res) } return rb.getManifestV21(dockerURL, res) @@ -405,12 +399,7 @@ func (rb *RepositoryBackend) getConfigV22(dockerURL *common.ParsedDockerURL, con rb.setBasicAuth(req) - accepting := []string{ - typesV2.MediaTypeOCIConfig, - typesV2.MediaTypeDockerV22Config, - } - - res, err := rb.makeRequest(req, dockerURL.ImageName, accepting) + res, err := rb.makeRequest(req, dockerURL.ImageName, rb.mediaTypes.ConfigMediaTypes()) if err != nil { return err } @@ -491,12 +480,7 @@ func (rb *RepositoryBackend) getLayerV2(layerID string, dockerURL *common.Parsed rb.setBasicAuth(req) - accepting := []string{ - typesV2.MediaTypeDockerV22RootFS, - typesV2.MediaTypeOCILayer, - } - - res, err = rb.makeRequest(req, dockerURL.ImageName, accepting) + res, err = rb.makeRequest(req, dockerURL.ImageName, rb.mediaTypes.LayerMediaTypes()) if err != nil { return nil, nil, err } @@ -516,7 +500,7 @@ func (rb *RepositoryBackend) getLayerV2(layerID string, dockerURL *common.Parsed } res.Body.Close() res = nil - res, err = rb.makeRequest(req, dockerURL.ImageName, accepting) + res, err = rb.makeRequest(req, dockerURL.ImageName, rb.mediaTypes.LayerMediaTypes()) if err != nil { return nil, nil, err } diff --git a/lib/internal/typesV2/docker_types.go b/lib/internal/typesV2/docker_types.go index f8305f75..8d7f6fb1 100644 --- a/lib/internal/typesV2/docker_types.go +++ b/lib/internal/typesV2/docker_types.go @@ -18,23 +18,7 @@ import ( "encoding/json" "errors" - spec "github.com/opencontainers/image-spec/specs-go/v1" -) - -const ( - MediaTypeDockerV21Manifest = "application/vnd.docker.distribution.manifest.v1+json" - MediaTypeDockerV21SignedManifest = "application/vnd.docker.distribution.manifest.v1+prettyjws" - MediaTypeDockerV21ManifestLayer = "application/vnd.docker.container.image.rootfs.diff+x-gtar" - - MediaTypeDockerV22Manifest = "application/vnd.docker.distribution.manifest.v2+json" - MediaTypeDockerV22ManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" - MediaTypeDockerV22Config = "application/vnd.docker.container.image.v1+json" - MediaTypeDockerV22RootFS = "application/vnd.docker.image.rootfs.diff.tar.gzip" - - MediaTypeOCIManifest = spec.MediaTypeImageManifest - MediaTypeOCIManifestList = spec.MediaTypeImageManifestList - MediaTypeOCIConfig = spec.MediaTypeImageConfig - MediaTypeOCILayer = spec.MediaTypeImageLayer + "github.com/appc/docker2aci/lib/common" ) var ( @@ -74,7 +58,7 @@ func (im *ImageManifest) PrettyString() string { } func (im *ImageManifest) Validate() error { - if im.MediaType != MediaTypeDockerV22Manifest && im.MediaType != MediaTypeOCIManifest { + if im.MediaType != common.MediaTypeDockerV22Manifest && im.MediaType != common.MediaTypeOCIV1Manifest { return ErrIncorrectMediaType } if im.Config == nil { diff --git a/lib/tests/common.go b/lib/tests/common.go index fd1387a8..b5be72a4 100644 --- a/lib/tests/common.go +++ b/lib/tests/common.go @@ -10,6 +10,7 @@ import ( "os" "path" + "github.com/appc/docker2aci/lib/common" "github.com/appc/docker2aci/lib/internal/typesV2" ) @@ -105,9 +106,9 @@ func GenDocker22Manifest(destPath, configHash string, layerHashes []string) erro manifest := &typesV2.ImageManifest{ SchemaVersion: 2, - MediaType: typesV2.MediaTypeDockerV22Manifest, + MediaType: common.MediaTypeDockerV22Manifest, Config: &typesV2.ImageManifestDigest{ - MediaType: typesV2.MediaTypeDockerV22Config, + MediaType: common.MediaTypeDockerV22Config, Size: int(configSize), Digest: "sha256:" + configHash, }, @@ -119,7 +120,7 @@ func GenDocker22Manifest(destPath, configHash string, layerHashes []string) erro } manifest.Layers = append(manifest.Layers, &typesV2.ImageManifestDigest{ - MediaType: typesV2.MediaTypeDockerV22RootFS, + MediaType: common.MediaTypeDockerV22RootFS, Size: int(layerSize), Digest: "sha256:" + h, }) diff --git a/lib/tests/v22_test.go b/lib/tests/v22_test.go index fe670c15..da63537d 100644 --- a/lib/tests/v22_test.go +++ b/lib/tests/v22_test.go @@ -230,7 +230,7 @@ func TestFetchingByTagV22(t *testing.T) { } imgName := "docker2aci/dockerv22test" imgRef := "v0.1.0" - server := RunDockerRegistry(t, tmpDir, imgName, imgRef, typesV2.MediaTypeDockerV22Manifest) + server := RunDockerRegistry(t, tmpDir, imgName, imgRef, d2acommon.MediaTypeDockerV22Manifest) defer server.Close() bareServerURL := strings.TrimPrefix(server.URL, "http://") @@ -347,7 +347,7 @@ func TestFetchingByDigestV22(t *testing.T) { } imgName := "docker2aci/dockerv22test" imgRef := "sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2" - server := RunDockerRegistry(t, tmpDir, imgName, imgRef, typesV2.MediaTypeDockerV22Manifest) + server := RunDockerRegistry(t, tmpDir, imgName, imgRef, d2acommon.MediaTypeDockerV22Manifest) defer server.Close() localUrl := path.Join(strings.TrimPrefix(server.URL, "http://"), imgName) + "@" + imgRef @@ -396,7 +396,7 @@ func TestFetchingMultipleLayersV22(t *testing.T) { } imgName := "docker2aci/dockerv22test" imgRef := "v0.1.0" - server := RunDockerRegistry(t, tmpDir, imgName, imgRef, typesV2.MediaTypeDockerV22Manifest) + server := RunDockerRegistry(t, tmpDir, imgName, imgRef, d2acommon.MediaTypeDockerV22Manifest) defer server.Close() localUrl := path.Join(strings.TrimPrefix(server.URL, "http://"), imgName) + ":" + imgRef diff --git a/tests/test.sh b/tests/test.sh index 27567542..52526210 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -21,6 +21,7 @@ go vet ./pkg/... go vet ./lib/... go test -v ${REPO_PATH}/lib/tests go test -v ${REPO_PATH}/lib/internal +go test -v ${REPO_PATH}/lib/common DOCKER2ACI=../bin/docker2aci PREFIX=docker2aci-tests