diff --git a/test/e2e/pull_chunked_test.go b/test/e2e/pull_chunked_test.go new file mode 100644 index 0000000000..4f164978e1 --- /dev/null +++ b/test/e2e/pull_chunked_test.go @@ -0,0 +1,250 @@ +//go:build linux || freebsd + +package integration + +import ( + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + + . "github.com/containers/podman/v5/test/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gexec" + "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +func pullChunkedTests() { // included in pull_test.go + if isRootless() { + err := podmanTest.RestoreArtifact(REGISTRY_IMAGE) + Expect(err).ToNot(HaveOccurred()) + } + lock := GetPortLock(pullChunkedRegistryPort) + defer lock.Unlock() + + pullChunkedRegistryPrefix := "docker://localhost:" + pullChunkedRegistryPort + "/" + + imageDir := filepath.Join(tempdir, "images") + err := os.MkdirAll(imageDir, 0o700) + Expect(err).NotTo(HaveOccurred()) + + // Prepare test images. + pullChunkedStartRegistry() + chunkedNormal := &pullChunkedTestImage{ + registryRef: pullChunkedRegistryPrefix + "chunked-normal", + dirPath: filepath.Join(imageDir, "chunked-normal"), + } + podmanTest.PodmanExitCleanly("pull", "-q", ALPINE) + podmanTest.PodmanExitCleanly("push", "-q", "--tls-verify=false", "--force-compression", "--compression-format", "zstd:chunked", ALPINE, chunkedNormal.registryRef) + skopeo := SystemExec("skopeo", []string{"copy", "-q", "--preserve-digests", "--all", "--src-tls-verify=false", chunkedNormal.registryRef, "dir:" + chunkedNormal.dirPath}) + skopeo.WaitWithDefaultTimeout() + Expect(skopeo).Should(ExitCleanly()) + jq := SystemExec("jq", []string{"-r", ".config.digest", filepath.Join(chunkedNormal.dirPath, "manifest.json")}) + jq.WaitWithDefaultTimeout() + Expect(jq).Should(ExitCleanly()) + cd, err := digest.Parse(jq.OutputToString()) + Expect(err).NotTo(HaveOccurred()) + chunkedNormal.configDigest = cd + + pullChunkedCreateImage := func(name string, editDiffIDs func([]digest.Digest) []digest.Digest) *pullChunkedTestImage { + res := pullChunkedTestImage{ + registryRef: pullChunkedRegistryPrefix + name, + dirPath: filepath.Join(imageDir, name), + } + cmd := SystemExec("cp", []string{"-a", chunkedNormal.dirPath, res.dirPath}) + cmd.WaitWithDefaultTimeout() + Expect(cmd).Should(ExitCleanly()) + + configBytes, err := os.ReadFile(filepath.Join(chunkedNormal.dirPath, chunkedNormal.configDigest.Encoded())) + Expect(err).NotTo(HaveOccurred()) + configBytes = editJSON(configBytes, func(config *imgspecv1.Image) { + config.RootFS.DiffIDs = editDiffIDs(config.RootFS.DiffIDs) + }) + res.configDigest = digest.FromBytes(configBytes) + err = os.WriteFile(filepath.Join(res.dirPath, res.configDigest.Encoded()), configBytes, 0o600) + Expect(err).NotTo(HaveOccurred()) + GinkgoWriter.Printf("CONFIG %q: %s\n\n", filepath.Join(res.dirPath, res.configDigest.Encoded()), string(configBytes)) + + manifestBytes, err := os.ReadFile(filepath.Join(chunkedNormal.dirPath, "manifest.json")) + Expect(err).NotTo(HaveOccurred()) + manifestBytes = editJSON(manifestBytes, func(manifest *imgspecv1.Manifest) { + manifest.Config.Digest = res.configDigest + manifest.Config.Size = int64(len(configBytes)) + }) + err = os.WriteFile(filepath.Join(res.dirPath, "manifest.json"), manifestBytes, 0o600) + Expect(err).NotTo(HaveOccurred()) + GinkgoWriter.Printf("MANIFEST %q: %s\n\n", filepath.Join(res.dirPath, "manifest.json"), string(manifestBytes)) + + res.push() + return &res + } + chunkedMismatch := pullChunkedCreateImage("mismatch", func(diffIDs []digest.Digest) []digest.Digest { + modified := slices.Clone(diffIDs) + digestBytes, err := hex.DecodeString(diffIDs[0].Encoded()) + Expect(err).NotTo(HaveOccurred()) + digestBytes[len(digestBytes)-1] ^= 1 + modified[0] = digest.NewDigestFromEncoded(diffIDs[0].Algorithm(), hex.EncodeToString(digestBytes)) + return modified + }) + chunkedMissing := pullChunkedCreateImage("missing", func(diffIDs []digest.Digest) []digest.Digest { + return nil + }) + chunkedEmpty := pullChunkedCreateImage("empty", func(diffIDs []digest.Digest) []digest.Digest { + res := make([]digest.Digest, len(diffIDs)) + for i := range res { + res[i] = "" + } + return res + }) + pullChunkedStopRegistry() + + // The actual test + for _, c := range []struct { + img *pullChunkedTestImage + fresh pullChunkedExpectation + reuse pullChunkedExpectation + onSuccess []string + onFailure []string + }{ + { + img: chunkedNormal, // FIXME + fresh: pullChunkedExpectation{ + onSuccess: []string{"Created zstd:chunked differ for blob"}, // Is a partial pull + }, + reuse: pullChunkedExpectation{ + onSuccess: []string{"Skipping blob .*already present"}, + }, + }, + { + img: chunkedMismatch, + fresh: pullChunkedExpectation{ + onFailure: []string{ + "Created zstd:chunked differ for blob", // Is a partial pull + "partial pull of blob.*uncompressed digest of layer.*is.*config claims", + }, + }, + reuse: pullChunkedExpectation{ + onFailure: []string{ + "trying to reuse blob.*layer.*does not match config's DiffID", + }, + }, + }, + { + img: chunkedMissing, + fresh: pullChunkedExpectation{ + onSuccess: []string{ + "Failed to retrieve partial blob: DiffID value for layer .* is unknown or explicitly empty", // Partial pull rejected + "Detected compression format zstd", // Non-partial pull happens + }, + }, + reuse: pullChunkedExpectation{ + onSuccess: []string{ + "Not using TOC .* to look for layer reuse: DiffID value for layer .* is unknown or explicitly empty", // Partial pull reuse rejected + "Skipping blob .*already present", // Non-partial reuse happens + }, + }, + }, { + img: chunkedEmpty, + fresh: pullChunkedExpectation{ + onSuccess: []string{}, + onFailure: []string{}, + }, + reuse: pullChunkedExpectation{ + onSuccess: []string{}, + onFailure: []string{}, + }, + }, + } { + GinkgoWriter.Printf("===\n===Testing %s\n===\n", c.img.registryRef) + + // Do each test with a clean slate: no layer metadata known, no blob info cache. + // Annoyingly, we have to re-start and re-populate the registry as well. + podmanTest.PodmanExitCleanly("system", "reset", "-f") + + pullChunkedStartRegistry() + c.img.push() + chunkedNormal.push() + + // Test fresh pull + c.fresh.testPull(c.img) + + podmanTest.PodmanExitCleanly("pull", "-q", "--tls-verify=false", chunkedNormal.registryRef) + + // Test pull after chunked layers are already known, to trigger the layer reuse code + c.reuse.testPull(c.img) + + pullChunkedStopRegistry() + } +} + +const pullChunkedRegistryPort = "5013" + +// pullChunkedStartRegistry creates a registry listening at pullChunkedRegistryPort within the current Podman environment. +func pullChunkedStartRegistry() { + podmanTest.PodmanExitCleanly("run", "-d", "--name", "registry", "--rm", "-p", pullChunkedRegistryPort+":5000", REGISTRY_IMAGE, "/entrypoint.sh", "/etc/docker/registry/config.yml") + if !WaitContainerReady(podmanTest, "registry", "listening on", 20, 1) { + Fail("Cannot start docker registry.") + } +} + +// pullChunkedStopRegistry stops a registry started by pullChunkedStartRegistry. +func pullChunkedStopRegistry() { + podmanTest.PodmanExitCleanly("stop", "registry") +} + +// pullChunkedTestImage centralizes data about a single test image in pullChunkedTests. +type pullChunkedTestImage struct { + registryRef, dirPath string + configDigest digest.Digest +} + +// push copies the image from dirPath to registryRef. +func (img *pullChunkedTestImage) push() { + skopeo := SystemExec("skopeo", []string{"copy", "-q", "--preserve-digests", "--all", "--dest-tls-verify=false", "dir:" + img.dirPath, img.registryRef}) + skopeo.WaitWithDefaultTimeout() + Expect(skopeo).Should(ExitCleanly()) +} + +// pullChunkedExpectations records the expected output of a single "podman pull" command. +type pullChunkedExpectation struct { + onSuccess []string // Expected debug log strings; should succeed if != nil + onFailure []string // Expected debug log strings; should fail if != nil +} + +// testPull performs one pull +func (expectation *pullChunkedExpectation) testPull(image *pullChunkedTestImage) { + session := podmanTest.Podman([]string{"--log-level=debug", "pull", "--tls-verify=false", image.registryRef}) + session.WaitWithDefaultTimeout() + GinkgoWriter.Printf("Pull exit code: %v\n", session.ExitCode()) + log := session.ErrorToString() + if expectation.onSuccess != nil { + Expect(session).Should(Exit(0)) + for _, s := range expectation.onSuccess { + Expect(regexp.MatchString(".*"+s+".*", log)).To(BeTrue(), s) + } + + s2 := podmanTest.PodmanExitCleanly("image", "inspect", "--format", "{{.ID}}", strings.TrimPrefix(image.registryRef, "docker://")) + Expect(s2.OutputToString()).Should(Equal(image.configDigest.Encoded())) + } else { + Expect(session).Should(Exit(125)) + for _, s := range expectation.onFailure { + Expect(regexp.MatchString(".*"+s+".*", log)).To(BeTrue(), s) + } + } +} + +// editJSON modifies a JSON-formatted input using the provided edit function. +func editJSON[T any](input []byte, edit func(*T)) []byte { + var value T + err = json.Unmarshal(input, &value) + Expect(err).NotTo(HaveOccurred()) + edit(&value) + res, err := json.Marshal(value) + Expect(err).NotTo(HaveOccurred()) + return res +} diff --git a/test/e2e/pull_test.go b/test/e2e/pull_test.go index ccc508df42..a3436b70b0 100644 --- a/test/e2e/pull_test.go +++ b/test/e2e/pull_test.go @@ -318,6 +318,8 @@ var _ = Describe("Podman pull", func() { Expect(session).Should(ExitCleanly()) }) + It("podman pull chunked images", pullChunkedTests) + It("podman pull from docker-archive", func() { SkipIfRemote("podman-remote does not support pulling from docker-archive")