From b1f0c7ad7b6e20bd5177de15ad2b58fe81355c02 Mon Sep 17 00:00:00 2001 From: 7h3-3mp7y-m4n Date: Tue, 29 Oct 2024 18:16:18 +0530 Subject: [PATCH 1/6] fixed OCI image import bug Signed-off-by: 7h3-3mp7y-m4n --- uriget/example_test.go | 26 +------------------------- uriget/uriget.go | 40 +++++++++++++++++++++++++++------------- 2 files changed, 28 insertions(+), 38 deletions(-) diff --git a/uriget/example_test.go b/uriget/example_test.go index f4ad784..fe08b98 100644 --- a/uriget/example_test.go +++ b/uriget/example_test.go @@ -52,31 +52,7 @@ func ExampleWithHttpClient() { // Output: failed to make get request: Get "https://example.com": no proxy } func ExampleGetFile_oci() { - testUrl := "oci://ghcr.io/score-spec/score-compose:0.18.0" - buff, err := GetFile(context.Background(), testUrl) - if err != nil { - fmt.Println("failed to pull OCI image:", err) - return - } - fmt.Println(len(buff) > 0) - // Output: - // true -} - -func ExampleGetFile_ociNoTag() { - testUrl := "oci://ghcr.io/score-spec/score-compose" - buff, err := GetFile(context.Background(), testUrl) - if err != nil { - fmt.Println("failed to pull OCI image:", err) - return - } - fmt.Println(len(buff) > 0) - // Output: - // true -} - -func ExampleGetFile_ociWithDigest() { - testUrl := "oci://ghcr.io/score-spec/score-compose@sha256:f3d8d5485a751cbdc91e073df1b6fbcde83f85a86ee3bc7d53e05b00452baedd" + testUrl := "oci://ghcr.io/score-spec/score-compose-community-provisioners:v0.1.0" buff, err := GetFile(context.Background(), testUrl) if err != nil { fmt.Println("failed to pull OCI image:", err) diff --git a/uriget/uriget.go b/uriget/uriget.go index 2d3a772..0bb05f4 100644 --- a/uriget/uriget.go +++ b/uriget/uriget.go @@ -6,6 +6,7 @@ package uriget import ( "context" + "encoding/json" "fmt" "io" "log" @@ -14,11 +15,11 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strings" "time" - "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content/oci" + v1 "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/registry" "oras.land/oras-go/v2/registry/remote" ) @@ -246,23 +247,36 @@ func (o *options) getOci(ctx context.Context, u *url.URL) ([]byte, error) { if err != nil { return nil, fmt.Errorf("can't parse artifact URL into a valid reference: %w", err) } - store, err := oci.New(o.tempDir) - if err != nil { - return nil, fmt.Errorf("failed to create OCI layout store: %w", err) + if ref.Reference == "" { + ref.Reference = "latest" } remoteRepo, err := remote.NewRepository(ref.String()) if err != nil { return nil, fmt.Errorf("failed to connect to remote repository: %w", err) } - tag := "latest" - if ref.Reference != "" { - tag = ref.Reference + _, rc, err := remoteRepo.Manifests().FetchReference(ctx, ref.Reference) + if err != nil { + return nil, fmt.Errorf("failed to fetch manifest: %w", err) + } + defer rc.Close() + var manifest v1.Manifest + if err = json.NewDecoder(rc).Decode(&manifest); err != nil { + return nil, fmt.Errorf("failed to decode manifest: %w", err) + } + index := slices.IndexFunc(manifest.Layers, func(descriptor v1.Descriptor) bool { + return strings.HasSuffix(descriptor.Annotations[v1.AnnotationTitle], ".yaml") + }) + if index < 0 { + return nil, fmt.Errorf("no .yaml file found in layers") } - manifestDescriptor, err := oras.Copy(ctx, remoteRepo, tag, store, tag, oras.DefaultCopyOptions) + _, rc, err = remoteRepo.Blobs().FetchReference(ctx, manifest.Layers[index].Digest.String()) if err != nil { - return nil, fmt.Errorf("failed to pull OCI image: %w", err) + return nil, fmt.Errorf("failed to fetch blob: %w", err) } - - o.logger.Printf("Pulled OCI image: %s with manifest descriptor: %v", u.String(), manifestDescriptor.Digest) - return []byte(manifestDescriptor.Digest), nil + defer rc.Close() + raw, err := io.ReadAll(rc) + if err != nil { + return nil, fmt.Errorf("failed to read blob content: %w", err) + } + return raw, nil } From 65416044c2b84bc8b5b63e78fb754ab2a5d50f87 Mon Sep 17 00:00:00 2001 From: 7h3-3mp7y-m4n Date: Wed, 30 Oct 2024 16:40:41 +0530 Subject: [PATCH 2/6] added the changes Signed-off-by: 7h3-3mp7y-m4n --- uriget/example_test.go | 2 +- uriget/uriget.go | 32 +++++++++++++++++++------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/uriget/example_test.go b/uriget/example_test.go index fe08b98..2e0dbd3 100644 --- a/uriget/example_test.go +++ b/uriget/example_test.go @@ -52,7 +52,7 @@ func ExampleWithHttpClient() { // Output: failed to make get request: Get "https://example.com": no proxy } func ExampleGetFile_oci() { - testUrl := "oci://ghcr.io/score-spec/score-compose-community-provisioners:v0.1.0" + testUrl := "oci://ghcr.io/score-spec/score-compose-community-provisioners:v0.1.0#00-service.provisioners.yaml" buff, err := GetFile(context.Background(), testUrl) if err != nil { fmt.Println("failed to pull OCI image:", err) diff --git a/uriget/uriget.go b/uriget/uriget.go index 0bb05f4..a428304 100644 --- a/uriget/uriget.go +++ b/uriget/uriget.go @@ -15,7 +15,6 @@ import ( "os" "os/exec" "path/filepath" - "slices" "strings" "time" @@ -245,38 +244,45 @@ func (o *options) getGit(ctx context.Context, u *url.URL) ([]byte, error) { func (o *options) getOci(ctx context.Context, u *url.URL) ([]byte, error) { ref, err := registry.ParseReference(u.Host + u.Path) if err != nil { - return nil, fmt.Errorf("can't parse artifact URL into a valid reference: %w", err) + return nil, fmt.Errorf("invalid artifact URL: %w", err) } if ref.Reference == "" { ref.Reference = "latest" } + specifiedFile := strings.TrimPrefix(u.Fragment, "#") remoteRepo, err := remote.NewRepository(ref.String()) if err != nil { - return nil, fmt.Errorf("failed to connect to remote repository: %w", err) + return nil, fmt.Errorf("connection to remote repository failed: %w", err) } _, rc, err := remoteRepo.Manifests().FetchReference(ctx, ref.Reference) if err != nil { - return nil, fmt.Errorf("failed to fetch manifest: %w", err) + return nil, fmt.Errorf("manifest fetch failed: %w", err) } defer rc.Close() var manifest v1.Manifest if err = json.NewDecoder(rc).Decode(&manifest); err != nil { - return nil, fmt.Errorf("failed to decode manifest: %w", err) + return nil, fmt.Errorf("manifest decode failed: %w", err) + } + var selectedLayer *v1.Descriptor + for _, layer := range manifest.Layers { + title := layer.Annotations[v1.AnnotationTitle] + if (specifiedFile != "" && title == specifiedFile) || + (specifiedFile == "" && strings.HasSuffix(title, ".yaml")) { + selectedLayer = &layer + break + } } - index := slices.IndexFunc(manifest.Layers, func(descriptor v1.Descriptor) bool { - return strings.HasSuffix(descriptor.Annotations[v1.AnnotationTitle], ".yaml") - }) - if index < 0 { - return nil, fmt.Errorf("no .yaml file found in layers") + if selectedLayer == nil { + return nil, fmt.Errorf("no matching .yaml file found in layers") } - _, rc, err = remoteRepo.Blobs().FetchReference(ctx, manifest.Layers[index].Digest.String()) + _, rc, err = remoteRepo.Blobs().FetchReference(ctx, selectedLayer.Digest.String()) if err != nil { - return nil, fmt.Errorf("failed to fetch blob: %w", err) + return nil, fmt.Errorf("blob fetch failed: %w", err) } defer rc.Close() raw, err := io.ReadAll(rc) if err != nil { - return nil, fmt.Errorf("failed to read blob content: %w", err) + return nil, fmt.Errorf("blob read failed: %w", err) } return raw, nil } From 22cfa8a19b53b77f628d0d58afb682bee74c5021 Mon Sep 17 00:00:00 2001 From: 7h3-3mp7y-m4n Date: Wed, 30 Oct 2024 18:59:03 +0530 Subject: [PATCH 3/6] added more testcases Signed-off-by: 7h3-3mp7y-m4n --- uriget/example_test.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/uriget/example_test.go b/uriget/example_test.go index 2e0dbd3..717025a 100644 --- a/uriget/example_test.go +++ b/uriget/example_test.go @@ -62,3 +62,39 @@ func ExampleGetFile_oci() { // Output: // true } + +func ExampleGetFiles_oci_git() { + ociTestUrl := "oci://ghcr.io/score-spec/score-compose-community-provisioners:v0.1.0#00-service.provisioners.yaml" + ociBuff, err := GetFile(context.Background(), ociTestUrl) + if err != nil { + fmt.Println("failed to pull OCI image:", err) + return + } + gitTestUrl := "git-https://github.com/score-spec/community-provisioners.git/score-compose/00-service.provisioners.yaml" + gitBuff, err := GetFile(context.Background(), gitTestUrl) + if err != nil { + fmt.Println("failed to pull file in git:", err) + return + } + fmt.Println(len(ociBuff) == len(gitBuff)) + // Output: + // true +} + +func ExampleGetFile_oci_https() { + ociTestUrl := "oci://ghcr.io/score-spec/score-compose-community-provisioners:v0.1.0#00-service.provisioners.yaml" + ociBuff, err := GetFile(context.Background(), ociTestUrl) + if err != nil { + fmt.Println("failed to pull OCI image:", err) + return + } + httpsTestUrl := "https://github.com/score-spec/community-provisioners/raw/v0.1.0/score-compose/00-service.provisioners.yaml" + httpsbuff, err := GetFile(context.Background(), httpsTestUrl) + if err != nil { + fmt.Println("failed to pull file by HTTPS:", err) + return + } + fmt.Println(len(ociBuff) == len(httpsbuff)) + // Output: + // true +} From 56de00d82c80ed2ed6680166c747a329c47f1e82 Mon Sep 17 00:00:00 2001 From: 7h3-3mp7y-m4n Date: Wed, 30 Oct 2024 19:03:26 +0530 Subject: [PATCH 4/6] fix the name convention for go vet Signed-off-by: 7h3-3mp7y-m4n --- uriget/example_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uriget/example_test.go b/uriget/example_test.go index 717025a..99c4dff 100644 --- a/uriget/example_test.go +++ b/uriget/example_test.go @@ -63,7 +63,7 @@ func ExampleGetFile_oci() { // true } -func ExampleGetFiles_oci_git() { +func ExampleGetFile_oci_git() { ociTestUrl := "oci://ghcr.io/score-spec/score-compose-community-provisioners:v0.1.0#00-service.provisioners.yaml" ociBuff, err := GetFile(context.Background(), ociTestUrl) if err != nil { From 14e5deb2d56a76f50561c4bbf53d5152f32c3f2c Mon Sep 17 00:00:00 2001 From: 7h3-3mp7y-m4n Date: Thu, 31 Oct 2024 00:42:44 +0530 Subject: [PATCH 5/6] Added the requested changes Signed-off-by: 7h3-3mp7y-m4n --- uriget/uriget.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/uriget/uriget.go b/uriget/uriget.go index a428304..caf5295 100644 --- a/uriget/uriget.go +++ b/uriget/uriget.go @@ -264,12 +264,18 @@ func (o *options) getOci(ctx context.Context, u *url.URL) ([]byte, error) { return nil, fmt.Errorf("manifest decode failed: %w", err) } var selectedLayer *v1.Descriptor + yamlFileCount := 0 for _, layer := range manifest.Layers { title := layer.Annotations[v1.AnnotationTitle] - if (specifiedFile != "" && title == specifiedFile) || - (specifiedFile == "" && strings.HasSuffix(title, ".yaml")) { - selectedLayer = &layer - break + if strings.HasSuffix(title, ".yaml") { + yamlFileCount++ + if specifiedFile == "" && yamlFileCount > 1 { + return nil, fmt.Errorf("manifest contains %d .yaml files; specify a specific file in the URL fragment", yamlFileCount) + } + if specifiedFile == "" || title == specifiedFile { + selectedLayer = &layer + break + } } } if selectedLayer == nil { @@ -280,7 +286,7 @@ func (o *options) getOci(ctx context.Context, u *url.URL) ([]byte, error) { return nil, fmt.Errorf("blob fetch failed: %w", err) } defer rc.Close() - raw, err := io.ReadAll(rc) + raw, err := readLimited(rc, o.limit) if err != nil { return nil, fmt.Errorf("blob read failed: %w", err) } From f743ee25746db6ab012095c3deaab9220d7c15d0 Mon Sep 17 00:00:00 2001 From: 7h3-3mp7y-m4n Date: Thu, 31 Oct 2024 00:48:25 +0530 Subject: [PATCH 6/6] Adding similarities like rest of function Signed-off-by: 7h3-3mp7y-m4n --- uriget/uriget.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uriget/uriget.go b/uriget/uriget.go index caf5295..d498571 100644 --- a/uriget/uriget.go +++ b/uriget/uriget.go @@ -286,9 +286,10 @@ func (o *options) getOci(ctx context.Context, u *url.URL) ([]byte, error) { return nil, fmt.Errorf("blob fetch failed: %w", err) } defer rc.Close() - raw, err := readLimited(rc, o.limit) + buff, err := readLimited(rc, o.limit) if err != nil { return nil, fmt.Errorf("blob read failed: %w", err) } - return raw, nil + o.logger.Printf("Read %d bytes from %s", len(buff), specifiedFile) + return buff, nil }