WIP tests

Signed-off-by: Miloslav Trmač <mitr@redhat.com>

test/e2e/pull_chunked_test.go

@@ -0,0 +1,250 @@
+//go:build linux || freebsd
+
+package integration
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strings"
+
+	. "github.com/containers/podman/v5/test/utils"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gexec"
+	"github.com/opencontainers/go-digest"
+	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+func pullChunkedTests() { // included in pull_test.go
+	if isRootless() {
+		err := podmanTest.RestoreArtifact(REGISTRY_IMAGE)
+		Expect(err).ToNot(HaveOccurred())
+	}
+	lock := GetPortLock(pullChunkedRegistryPort)
+	defer lock.Unlock()
+
+	pullChunkedRegistryPrefix := "docker://localhost:" + pullChunkedRegistryPort + "/"
+
+	imageDir := filepath.Join(tempdir, "images")
+	err := os.MkdirAll(imageDir, 0o700)
+	Expect(err).NotTo(HaveOccurred())
+
+	// Prepare test images.
+	pullChunkedStartRegistry()
+	chunkedNormal := &pullChunkedTestImage{
+		registryRef: pullChunkedRegistryPrefix + "chunked-normal",
+		dirPath:     filepath.Join(imageDir, "chunked-normal"),
+	}
+	podmanTest.PodmanExitCleanly("pull", "-q", ALPINE)
+	podmanTest.PodmanExitCleanly("push", "-q", "--tls-verify=false", "--force-compression", "--compression-format", "zstd:chunked", ALPINE, chunkedNormal.registryRef)
+	skopeo := SystemExec("skopeo", []string{"copy", "-q", "--preserve-digests", "--all", "--src-tls-verify=false", chunkedNormal.registryRef, "dir:" + chunkedNormal.dirPath})
+	skopeo.WaitWithDefaultTimeout()
+	Expect(skopeo).Should(ExitCleanly())
+	jq := SystemExec("jq", []string{"-r", ".config.digest", filepath.Join(chunkedNormal.dirPath, "manifest.json")})
+	jq.WaitWithDefaultTimeout()
+	Expect(jq).Should(ExitCleanly())
+	cd, err := digest.Parse(jq.OutputToString())
+	Expect(err).NotTo(HaveOccurred())
+	chunkedNormal.configDigest = cd
+
+	pullChunkedCreateImage := func(name string, editDiffIDs func([]digest.Digest) []digest.Digest) *pullChunkedTestImage {
+		res := pullChunkedTestImage{
+			registryRef: pullChunkedRegistryPrefix + name,
+			dirPath:     filepath.Join(imageDir, name),
+		}
+		cmd := SystemExec("cp", []string{"-a", chunkedNormal.dirPath, res.dirPath})
+		cmd.WaitWithDefaultTimeout()
+		Expect(cmd).Should(ExitCleanly())
+
+		configBytes, err := os.ReadFile(filepath.Join(chunkedNormal.dirPath, chunkedNormal.configDigest.Encoded()))
+		Expect(err).NotTo(HaveOccurred())
+		configBytes = editJSON(configBytes, func(config *imgspecv1.Image) {
+			config.RootFS.DiffIDs = editDiffIDs(config.RootFS.DiffIDs)
+		})
+		res.configDigest = digest.FromBytes(configBytes)
+		err = os.WriteFile(filepath.Join(res.dirPath, res.configDigest.Encoded()), configBytes, 0o600)
+		Expect(err).NotTo(HaveOccurred())
+		GinkgoWriter.Printf("CONFIG %q: %s\n\n", filepath.Join(res.dirPath, res.configDigest.Encoded()), string(configBytes))
+
+		manifestBytes, err := os.ReadFile(filepath.Join(chunkedNormal.dirPath, "manifest.json"))
+		Expect(err).NotTo(HaveOccurred())
+		manifestBytes = editJSON(manifestBytes, func(manifest *imgspecv1.Manifest) {
+			manifest.Config.Digest = res.configDigest
+			manifest.Config.Size = int64(len(configBytes))
+		})
+		err = os.WriteFile(filepath.Join(res.dirPath, "manifest.json"), manifestBytes, 0o600)
+		Expect(err).NotTo(HaveOccurred())
+		GinkgoWriter.Printf("MANIFEST %q: %s\n\n", filepath.Join(res.dirPath, "manifest.json"), string(manifestBytes))
+
+		res.push()
+		return &res
+	}
+	chunkedMismatch := pullChunkedCreateImage("mismatch", func(diffIDs []digest.Digest) []digest.Digest {
+		modified := slices.Clone(diffIDs)
+		digestBytes, err := hex.DecodeString(diffIDs[0].Encoded())
+		Expect(err).NotTo(HaveOccurred())
+		digestBytes[len(digestBytes)-1] ^= 1
+		modified[0] = digest.NewDigestFromEncoded(diffIDs[0].Algorithm(), hex.EncodeToString(digestBytes))
+		return modified
+	})
+	chunkedMissing := pullChunkedCreateImage("missing", func(diffIDs []digest.Digest) []digest.Digest {
+		return nil
+	})
+	chunkedEmpty := pullChunkedCreateImage("empty", func(diffIDs []digest.Digest) []digest.Digest {
+		res := make([]digest.Digest, len(diffIDs))
+		for i := range res {
+			res[i] = ""
+		}
+		return res
+	})
+	pullChunkedStopRegistry()
+
+	// The actual test
+	for _, c := range []struct {
+		img       *pullChunkedTestImage
+		fresh     pullChunkedExpectation
+		reuse     pullChunkedExpectation
+		onSuccess []string
+		onFailure []string
+	}{
+		{
+			img: chunkedNormal, // FIXME
+			fresh: pullChunkedExpectation{
+				onSuccess: []string{"Created zstd:chunked differ for blob"}, // Is a partial pull
+			},
+			reuse: pullChunkedExpectation{
+				onSuccess: []string{"Skipping blob .*already present"},
+			},
+		},
+		{
+			img: chunkedMismatch,
+			fresh: pullChunkedExpectation{
+				onFailure: []string{
+					"Created zstd:chunked differ for blob", // Is a partial pull
+					"partial pull of blob.*uncompressed digest of layer.*is.*config claims",
+				},
+			},
+			reuse: pullChunkedExpectation{
+				onFailure: []string{
+					"trying to reuse blob.*layer.*does not match config's DiffID",
+				},
+			},
+		},
+		{
+			img: chunkedMissing,
+			fresh: pullChunkedExpectation{
+				onSuccess: []string{
+					"Failed to retrieve partial blob: DiffID value for layer .* is unknown or explicitly empty", // Partial pull rejected
+					"Detected compression format zstd", // Non-partial pull happens
+				},
+			},
+			reuse: pullChunkedExpectation{
+				onSuccess: []string{
+					"Not using TOC .* to look for layer reuse: DiffID value for layer .* is unknown or explicitly empty", // Partial pull reuse rejected
+					"Skipping blob .*already present", // Non-partial reuse happens
+				},
+			},
+		}, {
+			img: chunkedEmpty,
+			fresh: pullChunkedExpectation{
+				onSuccess: []string{},
+				onFailure: []string{},
+			},
+			reuse: pullChunkedExpectation{
+				onSuccess: []string{},
+				onFailure: []string{},
+			},
+		},
+	} {
+		GinkgoWriter.Printf("===\n===Testing %s\n===\n", c.img.registryRef)
+
+		// Do each test with a clean slate: no layer metadata known, no blob info cache.
+		// Annoyingly, we have to re-start and re-populate the registry as well.
+		podmanTest.PodmanExitCleanly("system", "reset", "-f")
+
+		pullChunkedStartRegistry()
+		c.img.push()
+		chunkedNormal.push()
+
+		// Test fresh pull
+		c.fresh.testPull(c.img)
+
+		podmanTest.PodmanExitCleanly("pull", "-q", "--tls-verify=false", chunkedNormal.registryRef)
+
+		// Test pull after chunked layers are already known, to trigger the layer reuse code
+		c.reuse.testPull(c.img)
+
+		pullChunkedStopRegistry()
+	}
+}
+
+const pullChunkedRegistryPort = "5013"
+
+// pullChunkedStartRegistry creates a registry listening at pullChunkedRegistryPort within the current Podman environment.
+func pullChunkedStartRegistry() {
+	podmanTest.PodmanExitCleanly("run", "-d", "--name", "registry", "--rm", "-p", pullChunkedRegistryPort+":5000", REGISTRY_IMAGE, "/entrypoint.sh", "/etc/docker/registry/config.yml")
+	if !WaitContainerReady(podmanTest, "registry", "listening on", 20, 1) {
+		Fail("Cannot start docker registry.")
+	}
+}
+
+// pullChunkedStopRegistry stops a registry started by pullChunkedStartRegistry.
+func pullChunkedStopRegistry() {
+	podmanTest.PodmanExitCleanly("stop", "registry")
+}
+
+// pullChunkedTestImage centralizes data about a single test image in pullChunkedTests.
+type pullChunkedTestImage struct {
+	registryRef, dirPath string
+	configDigest         digest.Digest
+}
+
+// push copies the image from dirPath to registryRef.
+func (img *pullChunkedTestImage) push() {
+	skopeo := SystemExec("skopeo", []string{"copy", "-q", "--preserve-digests", "--all", "--dest-tls-verify=false", "dir:" + img.dirPath, img.registryRef})
+	skopeo.WaitWithDefaultTimeout()
+	Expect(skopeo).Should(ExitCleanly())
+}
+
+// pullChunkedExpectations records the expected output of a single "podman pull" command.
+type pullChunkedExpectation struct {
+	onSuccess []string // Expected debug log strings; should succeed if != nil
+	onFailure []string // Expected debug log strings; should fail if != nil
+}
+
+// testPull performs one pull
+func (expectation *pullChunkedExpectation) testPull(image *pullChunkedTestImage) {
+	session := podmanTest.Podman([]string{"--log-level=debug", "pull", "--tls-verify=false", image.registryRef})
+	session.WaitWithDefaultTimeout()
+	GinkgoWriter.Printf("Pull exit code: %v\n", session.ExitCode())
+	log := session.ErrorToString()
+	if expectation.onSuccess != nil {
+		Expect(session).Should(Exit(0))
+		for _, s := range expectation.onSuccess {
+			Expect(regexp.MatchString(".*"+s+".*", log)).To(BeTrue(), s)
+		}
+
+		s2 := podmanTest.PodmanExitCleanly("image", "inspect", "--format", "{{.ID}}", strings.TrimPrefix(image.registryRef, "docker://"))
+		Expect(s2.OutputToString()).Should(Equal(image.configDigest.Encoded()))
+	} else {
+		Expect(session).Should(Exit(125))
+		for _, s := range expectation.onFailure {
+			Expect(regexp.MatchString(".*"+s+".*", log)).To(BeTrue(), s)
+		}
+	}
+}
+
+// editJSON modifies a JSON-formatted input using the provided edit function.
+func editJSON[T any](input []byte, edit func(*T)) []byte {
+	var value T
+	err = json.Unmarshal(input, &value)
+	Expect(err).NotTo(HaveOccurred())
+	edit(&value)
+	res, err := json.Marshal(value)
+	Expect(err).NotTo(HaveOccurred())
+	return res
+}

test/e2e/pull_test.go

@@ -318,6 +318,8 @@ var _ = Describe("Podman pull", func() {
 		Expect(session).Should(ExitCleanly())
 	})
 
+	It("podman pull chunked images", pullChunkedTests)
+
 	It("podman pull from docker-archive", func() {
 		SkipIfRemote("podman-remote does not support pulling from docker-archive")