Support for building overlaybd images

Signed-off-by: liulanzheng <lanzheng.liulz@alibaba-inc.com>

cache/blobs.go

@@ -1,12 +1,18 @@
 package cache
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"maps"
 	"os"
+	"path"
 	"strconv"
 
+	obdlabel "github.com/containerd/accelerated-container-image/pkg/label"
+	obdcmd "github.com/containerd/accelerated-container-image/pkg/utils"
+	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/containerd/v2/core/diff"
 	"github.com/containerd/containerd/v2/core/leases"
 	"github.com/containerd/containerd/v2/core/mount"
@@ -108,7 +114,6 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
 				}()
 
 				compressorFunc, finalize := comp.Type.Compress(ctx, comp)
-				mediaType := comp.Type.MediaType()
 
 				var lowerRef *immutableRef
 				switch sr.kind() {
@@ -180,6 +185,21 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
 						enableOverlay = false
 					}
 				}
+
+				mediaType := comp.Type.MediaType()
+				if sr.cm.Snapshotter.Name() == "overlaybd" {
+					snStat, err := sr.cm.Snapshotter.Stat(ctx, sr.getSnapshotID())
+					if err != nil {
+						return nil, errors.Wrapf(err, "failed to Stat overlaybd")
+					}
+					if bdPath := snStat.Labels[obdlabel.LocalOverlayBDPath]; bdPath != "" {
+						if err := commitOverlayBD(ctx, sr, &desc); err != nil {
+							return nil, err
+						}
+						mediaType = desc.MediaType
+						enableOverlay = false
+					}
+				}
 				if enableOverlay {
 					computed, ok, err := sr.tryComputeOverlayBlob(ctx, lower, upper, mediaType, sr.ID(), compressorFunc)
 					if !ok || err != nil {
@@ -499,3 +519,50 @@ func ensureCompression(ctx context.Context, ref *immutableRef, comp compression.
 	}
 	return nil
 }
+
+func commitOverlayBD(ctx context.Context, sr *immutableRef, desc *ocispecs.Descriptor) error {
+	snStat, err := sr.cm.Snapshotter.Stat(ctx, sr.getSnapshotID())
+	if err != nil {
+		return errors.Wrapf(err, "failed to Stat overlaybd")
+	}
+	bdPath := snStat.Labels[obdlabel.LocalOverlayBDPath]
+	if bdPath == "" {
+		return errors.New("missing overlaybd path label")
+	}
+	dir := path.Dir(bdPath)
+	commitPath := path.Join(dir, "overlaybd.commit")
+	err = obdcmd.Commit(ctx, dir, dir, true, "-t", "-z", "-f")
+	if err != nil {
+		return errors.Wrapf(err, "failed to overlaybd-commit")
+	}
+	cw, err := sr.cm.ContentStore.Writer(ctx, content.WithRef(sr.ID()))
+	if err != nil {
+		return errors.Wrapf(err, "failed to open writer")
+	}
+	fi, err := os.Open(commitPath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open overlaybd commit file")
+	}
+	sz, err := io.Copy(cw, bufio.NewReader(fi))
+	if err != nil {
+		return errors.Wrapf(err, "failed to do io.Copy()")
+	}
+	dgst := cw.Digest()
+	labels := map[string]string{
+		labels.LabelUncompressed:     dgst.String(),
+		obdlabel.OverlayBDBlobDigest: dgst.String(),
+		obdlabel.OverlayBDBlobSize:   fmt.Sprintf("%d", sz),
+	}
+	err = cw.Commit(ctx, sz, dgst, content.WithLabels(labels))
+	if err != nil {
+		return errors.Wrapf(err, "failed to do cw.Commit")
+	}
+	desc.Digest = dgst
+	desc.Size = sz
+	desc.MediaType = ocispecs.MediaTypeImageLayer
+	desc.Annotations = map[string]string{
+		obdlabel.OverlayBDBlobDigest: string(desc.Digest),
+		obdlabel.OverlayBDBlobSize:   fmt.Sprintf("%d", desc.Size),
+	}
+	return nil
+}

cache/manager.go

@@ -8,9 +8,11 @@ import (
 	"sync"
 	"time"
 
+	obdlabel "github.com/containerd/accelerated-container-image/pkg/label"
 	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/containerd/v2/core/diff"
 	"github.com/containerd/containerd/v2/core/leases"
+	"github.com/containerd/containerd/v2/core/snapshots"
 	"github.com/containerd/containerd/v2/pkg/filters"
 	"github.com/containerd/containerd/v2/pkg/gc"
 	"github.com/containerd/containerd/v2/pkg/labels"
@@ -626,6 +628,10 @@ func (cm *cacheManager) New(ctx context.Context, s ImmutableRef, sess session.Gr
 		}); rerr != nil {
 			return nil, rerr
 		}
+	} else if cm.Snapshotter.Name() == "overlaybd" && parent != nil {
+		// Snapshotter will create a R/W block device directly as rootfs with this label
+		rwLabels := map[string]string{obdlabel.SupportReadWriteMode: "dev"}
+		err = cm.Snapshotter.Prepare(ctx, snapshotID, parentSnapshotID, snapshots.WithLabels(rwLabels))
 	} else {
 		err = cm.Snapshotter.Prepare(ctx, snapshotID, parentSnapshotID)
 	}

cache/refs.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	obdlabel "github.com/containerd/accelerated-container-image/pkg/label"
 	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/containerd/v2/core/images"
 	"github.com/containerd/containerd/v2/core/leases"
@@ -41,7 +42,7 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-var additionalAnnotations = append(compression.EStargzAnnotations, labels.LabelUncompressed)
+var additionalAnnotations = append(append(compression.EStargzAnnotations, obdlabel.OverlayBDAnnotations...), labels.LabelUncompressed)
 
 // Ref is a reference to cacheable objects.
 type Ref interface {
@@ -419,7 +420,11 @@ func (cr *cacheRecord) mount(ctx context.Context) (_ snapshot.Mountable, rerr er
 		// Return the mount direct from View rather than setting it using the Mounts call below.
 		// The two are equivalent for containerd snapshotters but the moby snapshotter requires
 		// the use of the mountable returned by View in this case.
-		mnts, err := cr.cm.Snapshotter.View(ctx, mountSnapshotID, cr.getSnapshotID())
+		labels := make(map[string]string)
+		if cr.cm.Snapshotter.Name() == "overlaybd" {
+			labels["containerd.io/snapshot/overlaybd.writable"] = "dev"
+		}
+		mnts, err := cr.cm.Snapshotter.View(ctx, mountSnapshotID, cr.getSnapshotID(), snapshots.WithLabels(labels))
 		if err != nil && !cerrdefs.IsAlreadyExists(err) {
 			return nil, err
 		}
@@ -1014,6 +1019,10 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro
 			return err
 		}
 		return rerr
+	} else if sr.cm.Snapshotter.Name() == "overlaybd" {
+		if rerr = sr.prepareRemoteSnapshotsOverlaybdMode(ctx); rerr == nil {
+			return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false)
+		}
 	}
 
 	return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false)
@@ -1141,6 +1150,55 @@ func (sr *immutableRef) prepareRemoteSnapshotsStargzMode(ctx context.Context, s
 	return err
 }
 
+func (sr *immutableRef) prepareRemoteSnapshotsOverlaybdMode(ctx context.Context) error {
+	_, err := g.Do(ctx, sr.ID()+"-prepare-remote-snapshot", func(ctx context.Context) (_ *leaseutil.LeaseRef, rerr error) {
+		dhs := sr.descHandlers
+		for _, r := range sr.layerChain() {
+			r := r
+			snapshotID := r.getSnapshotID()
+			if _, err := r.cm.Snapshotter.Stat(ctx, snapshotID); err == nil {
+				continue
+			}
+			dh := dhs[digest.Digest(r.getBlob())]
+			if dh == nil {
+				// We cannot prepare remote snapshots without descHandler.
+				return nil, nil
+			}
+			defaultLabels := snapshots.FilterInheritedLabels(dh.SnapshotLabels)
+			if defaultLabels == nil {
+				defaultLabels = make(map[string]string)
+			}
+			defaultLabels["containerd.io/snapshot.ref"] = snapshotID
+			// Prepare remote snapshots
+			var (
+				key  = fmt.Sprintf("tmp-%s %s", identity.NewID(), r.getChainID())
+				opts = []snapshots.Opt{
+					snapshots.WithLabels(defaultLabels),
+				}
+			)
+			parentID := ""
+			if r.layerParent != nil {
+				parentID = r.layerParent.getSnapshotID()
+			}
+			if err := r.cm.Snapshotter.Prepare(ctx, key, parentID, opts...); err != nil {
+				if cerrdefs.IsAlreadyExists(err) {
+					// Check if the targeting snapshot ID has been prepared as
+					// a remote snapshot in the snapshotter.
+					_, err := r.cm.Snapshotter.Stat(ctx, snapshotID)
+					if err == nil { // usable as remote snapshot without unlazying.
+						// Try the next layer as well.
+						continue
+					}
+				}
+			}
+			// This layer and all upper layers cannot be prepared without unlazying.
+			break
+		}
+		return nil, nil
+	})
+	return err
+}
+
 func makeTmpLabelsStargzMode(labels map[string]string, s session.Group) (fields []string, res map[string]string) {
 	res = make(map[string]string)
 	// Append unique ID to labels for avoiding collision of labels among calls
@@ -1310,7 +1368,12 @@ func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg pr
 
 	key := fmt.Sprintf("extract-%s %s", identity.NewID(), sr.getChainID())
 
-	err = sr.cm.Snapshotter.Prepare(ctx, key, parentID)
+	if sr.cm.Snapshotter.Name() == "overlaybd" {
+		err = sr.cm.Snapshotter.Prepare(ctx, key, parentID,
+			snapshots.WithLabels(map[string]string{"containerd.io/snapshot.ref": string(desc.Digest)}))
+	} else {
+		err = sr.cm.Snapshotter.Prepare(ctx, key, parentID)
+	}
 	if err != nil {
 		return err
 	}

cache/remotecache/registry/registry.go

@@ -165,6 +165,7 @@ func (dsl *withDistributionSourceLabel) SnapshotLabels(descs []ocispecs.Descript
 		labels = make(map[string]string)
 	}
 	maps.Copy(labels, estargz.SnapshotLabels(dsl.ref, descs, index))
+	labels["containerd.io/snapshot/image-ref"] = dsl.ref
 	return labels
 }
 

docs/overlaybd.md

@@ -0,0 +1,39 @@
+# Overlaybd
+
+[Overlaybd](https://github.com/containerd/overlaybd) is a novel layering block-level image format, which is design for container, secure container and applicable to virtual machine. And it is an open-source implementation of paper [DADI: Block-Level Image Service for Agile and Elastic Application Deployment. USENIX ATC'20"](https://www.usenix.org/conference/atc20/presentation/li-huiba).
+
+## Build Overlaybd Images
+
+Before building overlaybd images, ensure that `overlaybd-tcmu` and `overlaybd-snapshotter` are active by referring to the [QUICKSTART](https://github.com/containerd/accelerated-container-image/blob/main/docs/QUICKSTART.md#install) guide.
+
+To use buildkit to build overlaybd images, you should specify `--oci-worker-snapshotter=overlaybd` and `--oci-worker-proxy-snapshotter-path=/run/overlaybd-snapshotter/overlaybd.sock` when start buildkitd:
+
+```bash
+buildkitd  --oci-worker-snapshotter=overlaybd  --oci-worker-proxy-snapshotter-path=/run/overlaybd-snapshotter/overlaybd.sock
+```
+If an overlaybd image is used in the FROM instruction of a Dockerfile, the build will produce a new overlaybd image. It is essential to include `--oci-mediatypes=true` and `--compression=uncompressed` while running buildctl:
+
+```bash
+buildctl build ... \
+--output type=image,name=<new image name>,push=true,oci-mediatypes=true,compression=uncompressed
+```
+
+## Performance
+
+In our test case Dockerfile, we used a 5GB OCI image (and corresponding overlaybd format), wrote some new layers of identical size, and recorded the time cost of image pull (as **pull** in the table below), building all lines in Dockerfile (as **build**), and exporting to image and pushing (as **push**).
+
+OCI:
+
+| **size per layer** | **layers** | **pull** | **build** | **push** | **total**  |
+| -------- | ---- | ---- | ----- | ---- | ---- |
+| 4GB      | 1    | 105.7| 23.5  | 219.4| 348.6|
+| 1GB      | 4    | 88.5 | 34.0  | 123.8| 246.3|
+| 256MB    | 10   | 92.1 | 20.7  | 63.6 | 176.4|
+
+Overlaybd:
+
+| **size per layer** | **layers** | **pull** | **build** | **push** | **total**  |
+| -------- | ---- | ---- | ----- | ---- | ---- |
+| 4GB      | 1    | 0.9  | 21.5  | 166.2| 188.6|
+| 1GB      | 4    | 0.9	 | 24.9	 | 72.9 | 98.7 |
+| 256MB    | 10   | 0.7  | 18.4  | 48.9 | 68.0 |

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.27
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.8
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2
+	github.com/containerd/accelerated-container-image v1.2.3
 	github.com/containerd/console v1.0.4
 	github.com/containerd/containerd/api v1.8.0
 	github.com/containerd/containerd/v2 v2.0.2

go.sum

@@ -85,6 +85,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=
+github.com/containerd/accelerated-container-image v1.2.3 h1:tAIoP7Z7b2xGhb7QCM5Fa+2xqWfPqRmyi5lodbsGGRA=
+github.com/containerd/accelerated-container-image v1.2.3/go.mod h1:EvKVWor6ZQNUyYp0MZm5hw4k21ropuz7EegM+m/Jb/Q=
 github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0=
 github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0=
 github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro=

source/containerimage/pull.go

@@ -152,6 +152,7 @@ func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cach
 					labels = make(map[string]string)
 				}
 				maps.Copy(labels, estargz.SnapshotLabels(p.manifest.Ref, p.manifest.Descriptors, i))
+				labels["containerd.io/snapshot/image-ref"] = p.manifest.Ref
 
 				p.descHandlers[desc.Digest] = &cache.DescHandler{
 					Provider:       p.manifest.Provider,