pre-install-payload: Handle the nydus-snapshotter

nydus-snapshotter / nydus will be used to get rid of the containerd fork
we have, allowing us to do both the image pulling on the host side and
inside the guest.

NOTE:
This PR should NOT be merged as it's, as it breaks s390x payload build.

Signed-off-by: ChengyuZhu6 <chengyu.zhu@intel.com>
Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>

config/samples/ccruntime/base/ccruntime.yaml

@@ -55,6 +55,8 @@ spec:
           name: confidential-containers-artifacts
         - mountPath: /etc/systemd/system/
           name: etc-systemd-system
+        - mountPath: /etc/containerd/
+          name: containerd-conf
       volumes:
         - hostPath:
             path: /opt/confidential-containers/
@@ -64,6 +66,10 @@ spec:
             path: /etc/systemd/system/
             type: ""
           name: etc-systemd-system
+        - hostPath:
+            path: /etc/containerd/
+            type: ""
+          name: containerd-conf
       environmentVariables:
         # If set to true, this will install the CoCo fork of the containerd,
         # the one allowing images to be pulled inside the guest and has patches
@@ -80,13 +86,20 @@ spec:
         # default: false
         - name: "INSTALL_VFIO_GPU_CONTAINERD"
           value: "false"
+        # If set to true, this will install nydus-snapshotter and nydus-image
+        # on the node
+        # default: false
+        - name: "INSTALL_NYDUS_SNAPSHOTTER"
+          value: "false"
     preInstall:
       image: quay.io/confidential-containers/reqs-payload
       volumeMounts:
         - mountPath: /opt/confidential-containers/
           name: confidential-containers-artifacts
         - mountPath: /etc/systemd/system/
           name: etc-systemd-system
+        - mountPath: /etc/containerd/
+          name: containerd-conf
       volumes:
         - hostPath:
             path: /opt/confidential-containers/
@@ -96,6 +109,10 @@ spec:
             path: /etc/systemd/system/
             type: ""
           name: etc-systemd-system
+        - hostPath:
+            path: /etc/containerd/
+            type: ""
+          name: containerd-conf
       environmentVariables:
         # If set to true, this will install the CoCo fork of the containerd,
         # the one allowing images to be pulled inside the guest and has patches
@@ -111,6 +128,11 @@ spec:
         # default: false
         - name: "INSTALL_VFIO_GPU_CONTAINERD"
           value: "false"
+        # If set to true, this will install nydus-snapshotter and nydus-image
+        # on the node
+        # default: false
+        - name: "INSTALL_NYDUS_SNAPSHOTTER"
+          value: "false"
     environmentVariables:
       - name: NODE_NAME
         valueFrom:

config/samples/enclave-cc/base/ccruntime-enclave-cc.yaml

@@ -49,11 +49,17 @@ spec:
     postUninstall:
       image: quay.io/confidential-containers/reqs-payload
       volumeMounts:
+        - mountPath: /etc/containerd/
+          name: containerd-conf
         - mountPath: /opt/confidential-containers/
           name: confidential-containers-artifacts
         - mountPath: /etc/systemd/system/
           name: etc-systemd-system
       volumes:
+        - hostPath:
+            path: /etc/containerd/
+            type: ""
+          name: containerd-conf
         - hostPath:
             path: /opt/confidential-containers/
             type: DirectoryOrCreate
@@ -76,14 +82,25 @@ spec:
         # default: false
         - name: "INSTALL_VFIO_GPU_CONTAINERD"
           value: "false"
+        # If set to true, this will install nydus-snapshotter and nydus-image
+        # on the node
+        # default: false
+        - name: "INSTALL_NYDUS_SNAPSHOTTER"
+          value: "false"
     preInstall:
       image: quay.io/confidential-containers/reqs-payload
       volumeMounts:
+        - mountPath: /etc/containerd/
+          name: containerd-conf
         - mountPath: /opt/confidential-containers/
           name: confidential-containers-artifacts
         - mountPath: /etc/systemd/system/
           name: etc-systemd-system
       volumes:
+        - hostPath:
+            path: /etc/containerd/
+            type: ""
+          name: containerd-conf
         - hostPath:
             path: /opt/confidential-containers/
             type: DirectoryOrCreate
@@ -106,6 +123,11 @@ spec:
         # default: false
         - name: "INSTALL_VFIO_GPU_CONTAINERD"
           value: "false"
+        # If set to true, this will install nydus-snapshotter and nydus-image
+        # on the node
+        # default: false
+        - name: "INSTALL_NYDUS_SNAPSHOTTER"
+          value: "false"
     environmentVariables:
       - name: NODE_NAME
         valueFrom:

install/pre-install-payload/Dockerfile

@@ -54,6 +54,38 @@ RUN \
 	tar xvzpf containerd-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz -C ${NODE_DESTINATION} && \
 	rm containerd-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz
 
+#### Nydus snapshotter & nydus image
+
+FROM golang:1.19-alpine AS nydus-binary-downloader
+
+ARG ARCH
+ARG NYDUS_SNAPSHOTTER_VERSION
+ARG NYDUS_SNAPSHOTTER_REPO
+ARG NYDUS_REPO
+ARG NYDUS_VERSION
+
+ARG DESTINATION=/opt/confidential-containers-pre-install-artifacts
+ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers
+
+RUN mkdir -p ${NODE_DESTINATION}/bin && \
+    apk add --no-cache ca-certificates build-base git curl && \
+    git clone ${NYDUS_SNAPSHOTTER_REPO} -b ${NYDUS_SNAPSHOTTER_VERSION} /nydus-snapshotter && \
+    make -C /nydus-snapshotter && \
+    chmod +x /nydus-snapshotter/bin/containerd-nydus-grpc && \
+	chmod +x /nydus-snapshotter/bin/nydus-overlayfs && \
+    mv /nydus-snapshotter/bin/containerd-nydus-grpc ${NODE_DESTINATION}/bin && \
+	mv /nydus-snapshotter/bin/nydus-overlayfs ${NODE_DESTINATION}/bin && \
+    rm -rf /nydus-snapshotter 
+
+RUN if [ "${ARCH}" != "s390x" ]; then \
+    curl -fOL --progress-bar ${NYDUS_REPO}/releases/download/${NYDUS_VERSION}/nydus-static-${NYDUS_VERSION}-linux-${ARCH}.tgz && \
+    tar xvzpf nydus-static-${NYDUS_VERSION}-linux-${ARCH}.tgz -C / && \
+    chmod +x /nydus-static/nydus-image && \
+    mv /nydus-static/nydus-image ${NODE_DESTINATION}/bin && \
+    rm -rf /nydus-static /nydus-static-${NYDUS_VERSION}-linux-${ARCH}.tgz; \
+	fi
+
+RUN apk del build-base git curl
 
 #### kubectl
 
@@ -78,12 +110,17 @@ ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers
 ARG NODE_CONTAINERD_SYSTEMD_DESTINATION=${DESTINATION}/etc/systemd/system/containerd.service.d/
 
 ARG CONTAINERD_SYSTEMD_ARTIFACTS=./containerd/containerd-for-cc-override.conf
+ARG NYDUS_SNAPSHOTTER_ARTIFACTS=./remote-snapshotter/nydus-snapshotter
 
 COPY --from=coco-containerd-binary-downloader ${NODE_DESTINATION}/bin/containerd ${NODE_DESTINATION}/bin/coco-containerd
 COPY --from=official-containerd-binary-downloader ${NODE_DESTINATION}/bin/containerd ${NODE_DESTINATION}/bin/official-containerd
 COPY --from=vfio-gpu-containerd-binary-downloader ${NODE_DESTINATION}/bin/containerd ${NODE_DESTINATION}/bin/vfio-gpu-containerd
+
+COPY --from=nydus-binary-downloader ${NODE_DESTINATION}/bin/* ${NODE_DESTINATION}/bin/
+
 COPY --from=kubectl-binary-downloader /usr/bin/kubectl /usr/bin/kubectl
 COPY ${CONTAINERD_SYSTEMD_ARTIFACTS} ${NODE_CONTAINERD_SYSTEMD_DESTINATION}
+COPY ${NYDUS_SNAPSHOTTER_ARTIFACTS}/* ${NODE_DESTINATION}/share/nydus-snapshotter/
 
 ARG CONTAINER_ENGINE_ARTIFACTS=./scripts
-COPY ${CONTAINER_ENGINE_ARTIFACTS}/* ${DESTINATION}/scripts/
+COPY ${CONTAINER_ENGINE_ARTIFACTS}/* ${DESTINATION}/scripts/

install/pre-install-payload/Makefile

@@ -1,11 +1,15 @@
 COCO_CONTAINERD_VERSION = 1.6.8.2
 OFFICIAL_CONTAINERD_VERSION = 1.7.0
 VFIO_GPU_CONTAINERD_VERSION = 1.7.0.0
+NYDUS_SNAPSHOTTER_VERSION = v0.12.0
+NYDUS_VERSION= v2.2.3
 
 BASH = bash
 
 reqs-image:
 	coco_containerd_version=$(COCO_CONTAINERD_VERSION) \
 	official_containerd_version=$(OFFICIAL_CONTAINERD_VERSION) \
 	vfio_gpu_containerd_version=$(VFIO_GPU_CONTAINERD_VERSION) \
+	nydus_snapshotter_version=${NYDUS_SNAPSHOTTER_VERSION} \
+	nydus_version=${NYDUS_VERSION} \
 	$(BASH) -x payload.sh

install/pre-install-payload/payload.sh

@@ -12,6 +12,10 @@ official_containerd_repo=${official_containerd_repo:-"https://github.com/contain
 official_containerd_version=${official_containerd_version:-"1.7.0"}
 vfio_gpu_containerd_repo=${vfio_gpu_containerd_repo:-"https://github.com/confidential-containers/containerd"}
 vfio_gpu_containerd_version=${vfio_gpu_containerd_version:-"1.7.0.0"}
+nydus_snapshotter_repo=${nydus_snapshotter_repo:-"https://github.com/containerd/nydus-snapshotter"}
+nydus_snapshotter_version=${nydus_snapshotter_version:-"v0.12.0"}
+nydus_repo=${nydus_repo:-"https://github.com/dragonflyoss/image-service"}
+nydus_version=${nydus_version:-"v2.2.3"}
 containerd_dir="$(mktemp -d -t containerd-XXXXXXXXXX)/containerd"
 extra_docker_manifest_flags="${extra_docker_manifest_flags:-}"
 
@@ -66,6 +70,10 @@ function build_payload() {
 			--build-arg OFFICIAL_CONTAINERD_REPO="${official_containerd_repo}" \
 			--build-arg VFIO_GPU_CONTAINERD_VERSION="${vfio_gpu_containerd_version}" \
 			--build-arg VFIO_GPU_CONTAINERD_REPO="${vfio_gpu_containerd_repo}" \
+			--build-arg NYDUS_SNAPSHOTTER_VERSION="${nydus_snapshotter_version}" \
+			--build-arg NYDUS_SNAPSHOTTER_REPO="${nydus_snapshotter_repo}" \
+			--build-arg NYDUS_VERSION="${nydus_version}" \
+			--build-arg NYDUS_REPO="${nydus_repo}" \
 			-t "${registry}:${kernel_arch}-${tag}" \
 			--platform="${arch}" \
 			--load \

install/pre-install-payload/remote-snapshotter/nydus-snapshotter/config-coco-guest-pulling.toml

@@ -0,0 +1,15 @@
+version = 1
+
+# Snapshotter's own home directory where it stores and creates necessary resources
+root = "/var/lib/containerd-nydus"
+
+# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket
+address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+
+[daemon]
+# Enable proxy mode
+fs_driver = "proxy"
+
+[snapshot]
+# Insert Kata volume information to `Mount.Options`
+enable_kata_volume = true

install/pre-install-payload/remote-snapshotter/nydus-snapshotter/config-coco-host-sharing.toml

@@ -0,0 +1,40 @@
+
+version = 1
+# Snapshotter's own home directory where it stores and creates necessary resources
+root = "/var/lib/containerd-nydus"
+# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket
+address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+# No nydusd daemon needed
+daemon_mode = "none"
+
+[daemon]
+# Use `blockdev` for tarfs
+fs_driver = "blockdev"
+# Path to nydus-image binary
+nydusimage_path = "/opt/confidential-containers/bin/nydus-image"
+
+[remote]
+skip_ssl_verify = true
+
+[snapshot]
+# Insert Kata volume information to `Mount.Options`
+enable_kata_volume = true
+
+[experimental.tarfs]
+# Whether to enable nydus tarfs mode. Tarfs is supported by:
+# - The EROFS filesystem driver since Linux 6.4
+# - Nydus Image Service release v2.3
+enable_tarfs = true
+
+# Mount rafs on host by loopdev and EROFS
+mount_tarfs_on_host = false
+
+# Mode to export tarfs images:
+# - "none" or "": do not export tarfs
+# - "layer_verity_only": only generate disk verity information for a layer blob
+# - "image_verity_only": only generate disk verity information for all blobs of an image
+# - "layer_block": generate a raw block disk image with tarfs for a layer
+# - "image_block": generate a raw block disk image with tarfs for an image
+# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info
+# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info
+export_mode = "image_block_with_verity"