diff --git a/config/samples/ccruntime/base/ccruntime.yaml b/config/samples/ccruntime/base/ccruntime.yaml index d209c5c7..40d75ae0 100644 --- a/config/samples/ccruntime/base/ccruntime.yaml +++ b/config/samples/ccruntime/base/ccruntime.yaml @@ -55,6 +55,8 @@ spec: name: confidential-containers-artifacts - mountPath: /etc/systemd/system/ name: etc-systemd-system + - mountPath: /etc/containerd/ + name: containerd-conf volumes: - hostPath: path: /opt/confidential-containers/ @@ -64,6 +66,10 @@ spec: path: /etc/systemd/system/ type: "" name: etc-systemd-system + - hostPath: + path: /etc/containerd/ + type: "" + name: containerd-conf environmentVariables: # If set to true, this will install the CoCo fork of the containerd, # the one allowing images to be pulled inside the guest and has patches @@ -80,6 +86,11 @@ spec: # default: false - name: "INSTALL_VFIO_GPU_CONTAINERD" value: "false" + # If set to true, this will install nydus-snapshotter and nydus-image + # on the node + # default: false + - name: "INSTALL_NYDUS_SNAPSHOTTER" + value: "false" preInstall: image: quay.io/confidential-containers/reqs-payload volumeMounts: @@ -87,6 +98,8 @@ spec: name: confidential-containers-artifacts - mountPath: /etc/systemd/system/ name: etc-systemd-system + - mountPath: /etc/containerd/ + name: containerd-conf volumes: - hostPath: path: /opt/confidential-containers/ @@ -96,6 +109,10 @@ spec: path: /etc/systemd/system/ type: "" name: etc-systemd-system + - hostPath: + path: /etc/containerd/ + type: "" + name: containerd-conf environmentVariables: # If set to true, this will install the CoCo fork of the containerd, # the one allowing images to be pulled inside the guest and has patches @@ -111,6 +128,11 @@ spec: # default: false - name: "INSTALL_VFIO_GPU_CONTAINERD" value: "false" + # If set to true, this will install nydus-snapshotter and nydus-image + # on the node + # default: false + - name: "INSTALL_NYDUS_SNAPSHOTTER" + value: "false" environmentVariables: - name: NODE_NAME valueFrom: diff --git a/config/samples/enclave-cc/base/ccruntime-enclave-cc.yaml b/config/samples/enclave-cc/base/ccruntime-enclave-cc.yaml index 772df519..72573dcc 100644 --- a/config/samples/enclave-cc/base/ccruntime-enclave-cc.yaml +++ b/config/samples/enclave-cc/base/ccruntime-enclave-cc.yaml @@ -49,11 +49,17 @@ spec: postUninstall: image: quay.io/confidential-containers/reqs-payload volumeMounts: + - mountPath: /etc/containerd/ + name: containerd-conf - mountPath: /opt/confidential-containers/ name: confidential-containers-artifacts - mountPath: /etc/systemd/system/ name: etc-systemd-system volumes: + - hostPath: + path: /etc/containerd/ + type: "" + name: containerd-conf - hostPath: path: /opt/confidential-containers/ type: DirectoryOrCreate @@ -76,14 +82,25 @@ spec: # default: false - name: "INSTALL_VFIO_GPU_CONTAINERD" value: "false" + # If set to true, this will install nydus-snapshotter and nydus-image + # on the node + # default: false + - name: "INSTALL_NYDUS_SNAPSHOTTER" + value: "false" preInstall: image: quay.io/confidential-containers/reqs-payload volumeMounts: + - mountPath: /etc/containerd/ + name: containerd-conf - mountPath: /opt/confidential-containers/ name: confidential-containers-artifacts - mountPath: /etc/systemd/system/ name: etc-systemd-system volumes: + - hostPath: + path: /etc/containerd/ + type: "" + name: containerd-conf - hostPath: path: /opt/confidential-containers/ type: DirectoryOrCreate @@ -106,6 +123,11 @@ spec: # default: false - name: "INSTALL_VFIO_GPU_CONTAINERD" value: "false" + # If set to true, this will install nydus-snapshotter and nydus-image + # on the node + # default: false + - name: "INSTALL_NYDUS_SNAPSHOTTER" + value: "false" environmentVariables: - name: NODE_NAME valueFrom: diff --git a/install/pre-install-payload/Dockerfile b/install/pre-install-payload/Dockerfile index eb60fa00..6b94bbb5 100644 --- a/install/pre-install-payload/Dockerfile +++ b/install/pre-install-payload/Dockerfile @@ -54,6 +54,40 @@ RUN \ tar xvzpf containerd-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz -C ${NODE_DESTINATION} && \ rm containerd-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz +#### Nydus snapshotter & nydus image + +FROM golang:1.19-alpine AS nydus-binary-downloader + +ARG ARCH +ARG NYDUS_SNAPSHOTTER_VERSION +ARG NYDUS_SNAPSHOTTER_REPO +ARG NYDUS_REPO +ARG NYDUS_VERSION + +ARG DESTINATION=/opt/confidential-containers-pre-install-artifacts +ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers + +ENV GOARCH=${ARCH} + +RUN mkdir -p ${NODE_DESTINATION}/bin && \ + apk add --no-cache ca-certificates build-base git curl && \ + git clone ${NYDUS_SNAPSHOTTER_REPO} -b ${NYDUS_SNAPSHOTTER_VERSION} /nydus-snapshotter && \ + make -C /nydus-snapshotter && \ + chmod +x /nydus-snapshotter/bin/containerd-nydus-grpc && \ + chmod +x /nydus-snapshotter/bin/nydus-overlayfs && \ + mv /nydus-snapshotter/bin/containerd-nydus-grpc ${NODE_DESTINATION}/bin && \ + mv /nydus-snapshotter/bin/nydus-overlayfs ${NODE_DESTINATION}/bin && \ + rm -rf /nydus-snapshotter + +RUN if [ "${ARCH}" != "s390x" ]; then \ + curl -fOL --progress-bar ${NYDUS_REPO}/releases/download/${NYDUS_VERSION}/nydus-static-${NYDUS_VERSION}-linux-${ARCH}.tgz && \ + tar xvzpf nydus-static-${NYDUS_VERSION}-linux-${ARCH}.tgz -C / && \ + chmod +x /nydus-static/nydus-image && \ + mv /nydus-static/nydus-image ${NODE_DESTINATION}/bin && \ + rm -rf /nydus-static /nydus-static-${NYDUS_VERSION}-linux-${ARCH}.tgz; \ + fi + +RUN apk del build-base git curl #### kubectl @@ -78,12 +112,17 @@ ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers ARG NODE_CONTAINERD_SYSTEMD_DESTINATION=${DESTINATION}/etc/systemd/system/containerd.service.d/ ARG CONTAINERD_SYSTEMD_ARTIFACTS=./containerd/containerd-for-cc-override.conf +ARG NYDUS_SNAPSHOTTER_ARTIFACTS=./remote-snapshotter/nydus-snapshotter COPY --from=coco-containerd-binary-downloader ${NODE_DESTINATION}/bin/containerd ${NODE_DESTINATION}/bin/coco-containerd COPY --from=official-containerd-binary-downloader ${NODE_DESTINATION}/bin/containerd ${NODE_DESTINATION}/bin/official-containerd COPY --from=vfio-gpu-containerd-binary-downloader ${NODE_DESTINATION}/bin/containerd ${NODE_DESTINATION}/bin/vfio-gpu-containerd + +COPY --from=nydus-binary-downloader ${NODE_DESTINATION}/bin/* ${NODE_DESTINATION}/bin/ + COPY --from=kubectl-binary-downloader /usr/bin/kubectl /usr/bin/kubectl COPY ${CONTAINERD_SYSTEMD_ARTIFACTS} ${NODE_CONTAINERD_SYSTEMD_DESTINATION} +COPY ${NYDUS_SNAPSHOTTER_ARTIFACTS}/* ${NODE_DESTINATION}/share/nydus-snapshotter/ ARG CONTAINER_ENGINE_ARTIFACTS=./scripts -COPY ${CONTAINER_ENGINE_ARTIFACTS}/* ${DESTINATION}/scripts/ +COPY ${CONTAINER_ENGINE_ARTIFACTS}/* ${DESTINATION}/scripts/ \ No newline at end of file diff --git a/install/pre-install-payload/Makefile b/install/pre-install-payload/Makefile index 9a9d980c..540e9dc6 100644 --- a/install/pre-install-payload/Makefile +++ b/install/pre-install-payload/Makefile @@ -1,6 +1,8 @@ COCO_CONTAINERD_VERSION = 1.6.8.2 OFFICIAL_CONTAINERD_VERSION = 1.7.0 VFIO_GPU_CONTAINERD_VERSION = 1.7.0.0 +NYDUS_SNAPSHOTTER_VERSION = v0.12.0 +NYDUS_VERSION= v2.2.3 BASH = bash @@ -8,4 +10,6 @@ reqs-image: coco_containerd_version=$(COCO_CONTAINERD_VERSION) \ official_containerd_version=$(OFFICIAL_CONTAINERD_VERSION) \ vfio_gpu_containerd_version=$(VFIO_GPU_CONTAINERD_VERSION) \ + nydus_snapshotter_version=${NYDUS_SNAPSHOTTER_VERSION} \ + nydus_version=${NYDUS_VERSION} \ $(BASH) -x payload.sh diff --git a/install/pre-install-payload/payload.sh b/install/pre-install-payload/payload.sh index 29ee1a8f..a3acb10d 100755 --- a/install/pre-install-payload/payload.sh +++ b/install/pre-install-payload/payload.sh @@ -12,6 +12,10 @@ official_containerd_repo=${official_containerd_repo:-"https://github.com/contain official_containerd_version=${official_containerd_version:-"1.7.0"} vfio_gpu_containerd_repo=${vfio_gpu_containerd_repo:-"https://github.com/confidential-containers/containerd"} vfio_gpu_containerd_version=${vfio_gpu_containerd_version:-"1.7.0.0"} +nydus_snapshotter_repo=${nydus_snapshotter_repo:-"https://github.com/containerd/nydus-snapshotter"} +nydus_snapshotter_version=${nydus_snapshotter_version:-"v0.13.0"} +nydus_repo=${nydus_repo:-"https://github.com/dragonflyoss/image-service"} +nydus_version=${nydus_version:-"v2.3.0-alpha.0"} containerd_dir="$(mktemp -d -t containerd-XXXXXXXXXX)/containerd" extra_docker_manifest_flags="${extra_docker_manifest_flags:-}" @@ -19,34 +23,33 @@ registry="${registry:-quay.io/confidential-containers/reqs-payload}" supported_arches=( "linux/amd64" - "linux/s390x" ) function setup_env_for_arch() { case "$1" in - "linux/amd64") - kernel_arch="x86_64" - golang_arch="amd64" - ;; - "linux/s390x") - kernel_arch="s390x" - golang_arch="s390x" - ;; - (*) echo "$1 is not supported" > /dev/stderr && exit 1 + "linux/amd64") + kernel_arch="x86_64" + golang_arch="amd64" + ;; + "linux/s390x") + kernel_arch="s390x" + golang_arch="s390x" + ;; + *) echo "$1 is not supported" >/dev/stderr && exit 1 ;; esac - + } function purge_previous_manifests() { manifest=${1} - + # We need to sanitise the name by: # * Replacing: # * '/' by '_' # * ':' by '-' - + sanitised_manifest="$(echo ${manifest} | sed 's|/|_|g' | sed 's|:|-|g')" - rm -rf ${HOME}/.docker/manifests/${sanitised_manifest} + rm -rf ${HOME}/.docker/manifests/${sanitised_manifest} || true } function build_payload() { @@ -66,6 +69,10 @@ function build_payload() { --build-arg OFFICIAL_CONTAINERD_REPO="${official_containerd_repo}" \ --build-arg VFIO_GPU_CONTAINERD_VERSION="${vfio_gpu_containerd_version}" \ --build-arg VFIO_GPU_CONTAINERD_REPO="${vfio_gpu_containerd_repo}" \ + --build-arg NYDUS_SNAPSHOTTER_VERSION="${nydus_snapshotter_version}" \ + --build-arg NYDUS_SNAPSHOTTER_REPO="${nydus_snapshotter_repo}" \ + --build-arg NYDUS_VERSION="${nydus_version}" \ + --build-arg NYDUS_REPO="${nydus_repo}" \ -t "${registry}:${kernel_arch}-${tag}" \ --platform="${arch}" \ --load \ @@ -73,7 +80,7 @@ function build_payload() { docker push "${registry}:${kernel_arch}-${tag}" done - purge_previous_manifests ${registry}:${tag} + purge_previous_manifests ${registry}:${tag} purge_previous_manifests ${registry}:latest docker manifest create ${extra_docker_manifest_flags} \ diff --git a/install/pre-install-payload/remote-snapshotter/nydus-snapshotter/config-coco-guest-pulling.toml b/install/pre-install-payload/remote-snapshotter/nydus-snapshotter/config-coco-guest-pulling.toml new file mode 100644 index 00000000..67a9be2d --- /dev/null +++ b/install/pre-install-payload/remote-snapshotter/nydus-snapshotter/config-coco-guest-pulling.toml @@ -0,0 +1,15 @@ +version = 1 + +# Snapshotter's own home directory where it stores and creates necessary resources +root = "/var/lib/containerd-nydus" + +# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket +address = "/run/containerd-nydus/containerd-nydus-grpc.sock" + +[daemon] +# Enable proxy mode +fs_driver = "proxy" + +[snapshot] +# Insert Kata volume information to `Mount.Options` +enable_kata_volume = true \ No newline at end of file diff --git a/install/pre-install-payload/remote-snapshotter/nydus-snapshotter/config-coco-host-sharing.toml b/install/pre-install-payload/remote-snapshotter/nydus-snapshotter/config-coco-host-sharing.toml new file mode 100644 index 00000000..fe82d175 --- /dev/null +++ b/install/pre-install-payload/remote-snapshotter/nydus-snapshotter/config-coco-host-sharing.toml @@ -0,0 +1,40 @@ + +version = 1 +# Snapshotter's own home directory where it stores and creates necessary resources +root = "/var/lib/containerd-nydus" +# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket +address = "/run/containerd-nydus/containerd-nydus-grpc.sock" +# No nydusd daemon needed +daemon_mode = "none" + +[daemon] +# Use `blockdev` for tarfs +fs_driver = "blockdev" +# Path to nydus-image binary +nydusimage_path = "/opt/confidential-containers/bin/nydus-image" + +[remote] +skip_ssl_verify = true + +[snapshot] +# Insert Kata volume information to `Mount.Options` +enable_kata_volume = true + +[experimental.tarfs] +# Whether to enable nydus tarfs mode. Tarfs is supported by: +# - The EROFS filesystem driver since Linux 6.4 +# - Nydus Image Service release v2.3 +enable_tarfs = true + +# Mount rafs on host by loopdev and EROFS +mount_tarfs_on_host = false + +# Mode to export tarfs images: +# - "none" or "": do not export tarfs +# - "layer_verity_only": only generate disk verity information for a layer blob +# - "image_verity_only": only generate disk verity information for all blobs of an image +# - "layer_block": generate a raw block disk image with tarfs for a layer +# - "image_block": generate a raw block disk image with tarfs for an image +# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info +# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info +export_mode = "image_block_with_verity" \ No newline at end of file diff --git a/install/pre-install-payload/scripts/reqs-deploy.sh b/install/pre-install-payload/scripts/reqs-deploy.sh index 75d591f7..448fa571 100755 --- a/install/pre-install-payload/scripts/reqs-deploy.sh +++ b/install/pre-install-payload/scripts/reqs-deploy.sh @@ -4,6 +4,8 @@ set -o errexit set -o pipefail set -o nounset +containerd_config="/etc/containerd/config.toml" + die() { msg="$*" echo "ERROR: $msg" >&2 @@ -20,7 +22,7 @@ function get_container_engine() { die "${container_engine} is not yet supported" fi - echo "$container_engine" + echo "$container_engine" } function set_container_engine() { @@ -38,7 +40,6 @@ function install_containerd_artefacts() { install -D -m 755 ${artifacts_dir}/opt/confidential-containers/bin/${flavour}-containerd /opt/confidential-containers/bin/containerd install -D -m 644 ${artifacts_dir}/etc/systemd/system/containerd.service.d/containerd-for-cc-override.conf /etc/systemd/system/containerd.service.d/containerd-for-cc-override.conf - } function install_coco_containerd_artefacts() { @@ -53,6 +54,23 @@ function install_vfio_gpu_containerd_artefacts() { install_containerd_artefacts "vfio-gpu" } +function install_nydus_snapshotter_artefacts() { + echo "Copying nydus-snapshotter artifacts onto host" + install -D -m 755 ${artifacts_dir}/opt/confidential-containers/bin/containerd-nydus-grpc /opt/confidential-containers/bin/containerd-nydus-grpc + install -D -m 755 ${artifacts_dir}/opt/confidential-containers/bin/nydus-overlayfs /opt/confidential-containers/bin/nydus-overlayfs + #NOTE: symlink nydus-overlayfs to /usr/local/bin or /usr/bin + ln -s /opt/confidential-containers/bin/nydus-overlayfs /usr/bin/nydus-overlayfs + if [ "$(uname -m)" != "s390x" ]; then + install -D -m 755 ${artifacts_dir}/opt/confidential-containers/bin/nydus-image /opt/confidential-containers/bin/nydus-image + fi + install -D -m 644 ${artifacts_dir}/opt/confidential-containers/share/nydus-snapshotter/config-coco-host-sharing.toml /opt/confidential-containers/share/nydus-snapshotter/config-coco-host-sharing.toml + install -D -m 644 ${artifacts_dir}/opt/confidential-containers/share/nydus-snapshotter/config-coco-host-sharing.toml /opt/confidential-containers/share/nydus-snapshotter/config-coco-host-sharing.toml + + configure_nydus_snapshotter_for_containerd + + restart_systemd_service +} + function install_artifacts() { if [ "${INSTALL_COCO_CONTAINERD}" = "true" ]; then install_coco_containerd_artefacts @@ -65,6 +83,10 @@ function install_artifacts() { if [ "${INSTALL_VFIO_GPU_CONTAINERD}" = "true" ]; then install_vfio_gpu_containerd_artefacts fi + + if [ "${INSTALL_NYDUS_SNAPSHOTTER}" = "true" ]; then + install_nydus_snapshotter_artefacts + fi } function uninstall_containerd_artefacts() { @@ -76,7 +98,7 @@ function uninstall_containerd_artefacts() { if [ -d /etc/systemd/system/${container_engine}.service.d ]; then rmdir --ignore-fail-on-non-empty /etc/systemd/system/${container_engine}.service.d fi - + restart_systemd_service echo "Removing the containerd binary" @@ -87,6 +109,24 @@ function uninstall_containerd_artefacts() { fi } +function uninstall_nydus_snapshotter_artefacts() { + remove_nydus_snapshotter_from_containerd + + restart_systemd_service + + echo "Removing nydus-snapshotter artifacts from host" + rm -f /opt/confidential-containers/bin/containerd-nydus-grpc + rm -f /opt/confidential-containers/bin/nydus-overlayfs + #NOTE: remove the link of nydus-overlayfs in /usr/local/bin or /usr/bin + rm /usr/bin/nydus-overlayfs + if [ "$(uname -m)" != "s390x" ]; then + rm -f /opt/confidential-containers/bin/nydus-image + fi + rm -f /opt/confidential-containers/share/remote-snapshotter/config-coco-host-sharing.toml + rm -f /opt/confidential-containers/share/remote-snapshotter/config-coco-guest-pulling.toml +} + + function uninstall_artifacts() { if [ "${INSTALL_COCO_CONTAINERD}" = "true" ] || [ "${INSTALL_OFFICIAL_CONTAINERD}" = "true" ] || [ "${INSTALL_VFIO_GPU_CONTAINERD}" = "true" ]; then uninstall_containerd_artefacts @@ -99,6 +139,33 @@ function restart_systemd_service() { host_systemctl restart "${container_engine}" } +function configure_nydus_snapshotter_for_containerd() { + echo "configure nydus snapshotter for containerd" + + if [ ! -f "$containerd_config" ]; then + die "failed to find containerd config" + fi + + if [ "${INSTALL_NYDUS_SNAPSHOTTER}" = "true" ]; then + echo "Plug nydus snapshotter into containerd" + snapshotter_socket="/run/containerd-nydus/containerd-nydus-grpc.sock" + fi + proxy_config=" [proxy_plugins.$SNAPSHOTTER]\n type = \"snapshot\"\n address = \"${snapshotter_socket}\"" + + if grep -q "\[proxy_plugins\]" "$containerd_config"; then + sed -i '/\[proxy_plugins\]/a\'"$proxy_config" "$containerd_config" + else + echo -e "[proxy_plugins]" >>"$containerd_config" + echo -e "$proxy_config" >>"$containerd_config" + fi +} + +function remove_nydus_snapshotter_from_containerd() { + echo "Remove nydus snapshotter from containerd" + + sed -i '/\[proxy_plugins.nydus\]/,/address = "\/run\/containerd-nydus\/containerd-nydus-grpc\.sock"/d' "$containerd_config" +} + label_node() { case "${1}" in install) @@ -120,6 +187,7 @@ function main() { echo "INSTALL_COCO_CONTAINERD: ${INSTALL_COCO_CONTAINERD}" echo "INSTALL_OFFICIAL_CONTAINERD: ${INSTALL_OFFICIAL_CONTAINERD}" echo "INSTALL_VFIO_GPU_CONTAINERD: ${INSTALL_VFIO_GPU_CONTAINERD}" + echo "INSTALL_NYDUS_SNAPSHOTTER: ${INSTALL_NYDUS_SNAPSHOTTER}" # script requires that user is root local euid=$(id -u)