Add bare-metal GPU runtime class

This adds a runtime class for the local just-based deployments as well as the release artifacts that corresponds to the GPU-enabled runtime for Contrast on bare-metal platforms.
edgelesssys · Jan 13, 2025 · fa64242 · fa64242
1 parent df4e08e
commit fa64242
Show file tree

Hide file tree

Showing 7 changed files with 136 additions and 94 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -247,6 +247,7 @@ jobs:
           coordinatorImg=$(nix run .#containers.push-coordinator -- "$container_registry/contrast/coordinator")
           nodeInstallerMsftImg=$(nix run .#containers.push-node-installer-microsoft -- "$container_registry/contrast/node-installer-microsoft")
           nodeInstallerKataImg=$(nix run .#containers.push-node-installer-kata -- "$container_registry/contrast/node-installer-kata")
+          nodeInstallerKataGPUImg=$(nix run .#containers.push-node-installer-kata-gpu -- "$container_registry/contrast/node-installer-kata")
           initializerImg=$(nix run .#containers.push-initializer -- "$container_registry/contrast/initializer")
           serviceMeshImg=$(nix run .#containers.push-service-mesh-proxy -- "$container_registry/contrast/service-mesh-proxy")
           tardevSnapshotterImg=$(nix run .#containers.push-tardev-snapshotter -- "$container_registry/contrast/tardev-snapshotter")
@@ -255,6 +256,7 @@ jobs:
           echo "coordinatorImg=$coordinatorImg" | tee -a "$GITHUB_ENV"
           echo "nodeInstallerMsftImg=$nodeInstallerMsftImg" | tee -a "$GITHUB_ENV"
           echo "nodeInstallerKataImg=$nodeInstallerKataImg" | tee -a "$GITHUB_ENV"
+          echo "nodeInstallerKataGPUImg=$nodeInstallerKataGPUImg" | tee -a "$GITHUB_ENV"
           echo "initializerImg=$initializerImg" | tee -a "$GITHUB_ENV"
           echo "serviceMeshImg=$serviceMeshImg" | tee -a "$GITHUB_ENV"
           echo "tardevSnapshotterImg=$tardevSnapshotterImg" | tee -a "$GITHUB_ENV"
@@ -270,6 +272,7 @@ jobs:
           echo "coordinatorImgTagged=$(tag "$coordinatorImg")" | tee -a "$GITHUB_ENV"
           echo "nodeInstallerMsftImgTagged=$(tag "$nodeInstallerMsftImg")" | tee -a "$GITHUB_ENV"
           echo "nodeInstallerKataImgTagged=$(tag "$nodeInstallerKataImg")" | tee -a "$GITHUB_ENV"
+          echo "nodeInstallerKataGPUImgTagged=$(tag "$nodeInstallerKataGPUImg")" | tee -a "$GITHUB_ENV"
           echo "initializerImgTagged=$(tag "$initializerImg")" | tee -a "$GITHUB_ENV"
           echo "serviceMeshImgTagged=$(tag "$serviceMeshImg")" | tee -a "$GITHUB_ENV"
           echo "cryptsetupImgTagged=$(tag "$cryptsetupImg")" | tee -a "$GITHUB_ENV"
@@ -291,6 +294,7 @@ jobs:
             echo "ghcr.io/edgelesssys/contrast/service-mesh-proxy:latest=$serviceMeshImgTagged"
             echo "ghcr.io/edgelesssys/contrast/node-installer-microsoft:latest=$nodeInstallerMsftImgTagged"
             echo "ghcr.io/edgelesssys/contrast/node-installer-kata:latest=$nodeInstallerKataImgTagged"
+            echo "ghcr.io/edgelesssys/contrast/node-installer-kata-gpu:latest=$nodeInstallerKataGPUImgTagged"
             echo "ghcr.io/edgelesssys/contrast/tardev-snapshotter:latest=$tardevSnapshotterImgTagged"
             echo "ghcr.io/edgelesssys/contrast/nydus-snapshotter:latest=$nydusSnapshotterImgTagged"
             echo "ghcr.io/edgelesssys/contrast/cryptsetup:latest=$cryptsetupImgTagged"

diff --git a/internal/kuberesource/parts.go b/internal/kuberesource/parts.go
@@ -128,6 +128,9 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 		snapshotterVolumes = tardevSnapshotterVolumes
 	case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.MetalQEMUSNPGPU:
 		nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest"
+		if platform == platforms.MetalQEMUSNPGPU {
+			nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata-gpu:latest"
+		}
 		snapshotter = nydusSnapshotter
 		nydusSnapshotterVolumes = append(nydusSnapshotterVolumes, Volume().
 			WithName("var-lib-containerd").
@@ -138,6 +141,9 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 		snapshotterVolumes = nydusSnapshotterVolumes
 	case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU, platforms.RKE2QEMUTDX:
 		nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest"
+		if platform == platforms.K3sQEMUSNPGPU {
+			nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata-gpu:latest"
+		}
 		snapshotter = nydusSnapshotter
 		nydusSnapshotterVolumes = append(nydusSnapshotterVolumes, Volume().
 			WithName("var-lib-containerd").

diff --git a/justfile b/justfile
@@ -47,10 +47,14 @@ node-installer platform=default_platform:
             just push "tardev-snapshotter"
             just push "node-installer-microsoft"
         ;;
-        "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"Metal-QEMU-SNP-GPU"|"K3s-QEMU-SNP"|"K3s-QEMU-SNP-GPU"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX")
+        "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX")
             just push "nydus-snapshotter"
             just push "node-installer-kata"
         ;;
+        "Metal-QEMU-SNP-GPU"|"K3s-QEMU-SNP-GPU")
+            just push "nydus-snapshotter"
+            just push "node-installer-kata-gpu"
+        ;;
         "AKS-PEER-SNP")
             nix run -L .#scripts.deploy-caa -- \
                 --kustomization=./infra/azure-peerpods/kustomization.yaml \

diff --git a/packages/by-name/kata/contrast-node-installer-image/package.nix b/packages/by-name/kata/contrast-node-installer-image/package.nix
@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 {
+  lib,
   ociLayerTar,
   ociImageManifest,
   ociImageLayout,
@@ -17,10 +18,16 @@
   OVMF-SNP,
   OVMF-TDX,
 
-  debugRuntime ? false,
+  debugRuntime ? true,
+  withGPU ? false,
 }:
 
 let
+  os-image = kata.kata-image.override {
+    inherit withGPU;
+    withDebug = debugRuntime;
+  };
+
   node-installer = ociLayerTar {
     files = [
       {
@@ -38,79 +45,82 @@ let
     files = [
       {
         source = writers.writeJSON "contrast-node-install.json" {
-          files = [
-            {
-              url = "file:///opt/edgeless/share/kata-containers.img";
-              path = "/opt/edgeless/@@runtimeName@@/share/kata-containers.img";
-            }
-            {
-              url = "file:///opt/edgeless/share/kata-kernel";
-              path = "/opt/edgeless/@@runtimeName@@/share/kata-kernel";
-            }
-            {
-              url = "file:///opt/edgeless/share/kata-initrd.zst";
-              path = "/opt/edgeless/@@runtimeName@@/share/kata-initrd.zst";
-            }
-            {
-              url = "file:///opt/edgeless/snp/bin/qemu-system-x86_64";
-              path = "/opt/edgeless/@@runtimeName@@/snp/bin/qemu-system-x86_64";
-              executable = true;
-            }
-            {
-              url = "file:///opt/edgeless/tdx/bin/qemu-system-x86_64";
-              path = "/opt/edgeless/@@runtimeName@@/tdx/bin/qemu-system-x86_64";
-              executable = true;
-            }
-            {
-              url = "file:///opt/edgeless/snp/share/OVMF.fd";
-              path = "/opt/edgeless/@@runtimeName@@/snp/share/OVMF.fd";
-            }
-            {
-              url = "file:///opt/edgeless/tdx/share/OVMF.fd";
-              path = "/opt/edgeless/@@runtimeName@@/tdx/share/OVMF.fd";
-            }
-            {
-              url = "file:///opt/edgeless/bin/containerd-shim-contrast-cc-v2";
-              path = "/opt/edgeless/@@runtimeName@@/bin/containerd-shim-contrast-cc-v2";
-              executable = true;
-            }
-            {
-              url = "file:///opt/edgeless/bin/kata-runtime";
-              path = "/opt/edgeless/@@runtimeName@@/bin/kata-runtime";
-              executable = true;
-            }
-            {
-              url = "file:///opt/edgeless/snp/share/qemu/kvmvapic.bin";
-              path = "/opt/edgeless/@@runtimeName@@/snp/share/qemu/kvmvapic.bin";
-            }
-            {
-              url = "file:///opt/edgeless/snp/share/qemu/linuxboot_dma.bin";
-              path = "/opt/edgeless/@@runtimeName@@/snp/share/qemu/linuxboot_dma.bin";
-            }
-            {
-              url = "file:///opt/edgeless/snp/share/qemu/efi-virtio.rom";
-              path = "/opt/edgeless/@@runtimeName@@/snp/share/qemu/efi-virtio.rom";
-            }
-            {
-              url = "file:///opt/edgeless/tdx/share/qemu/kvmvapic.bin";
-              path = "/opt/edgeless/@@runtimeName@@/tdx/share/qemu/kvmvapic.bin";
-            }
-            {
-              url = "file:///opt/edgeless/tdx/share/qemu/linuxboot_dma.bin";
-              path = "/opt/edgeless/@@runtimeName@@/tdx/share/qemu/linuxboot_dma.bin";
-            }
-            {
-              url = "file:///opt/edgeless/tdx/share/qemu/efi-virtio.rom";
-              path = "/opt/edgeless/@@runtimeName@@/tdx/share/qemu/efi-virtio.rom";
-            }
-            {
-              url = "file:///bin/nydus-overlayfs";
-              path = "/opt/edgeless/@@runtimeName@@/bin/nydus-overlayfs";
-              executable = true;
-            }
-          ];
+          files =
+            [
+              {
+                url = "file:///opt/edgeless/share/kata-containers.img";
+                path = "/opt/edgeless/@@runtimeName@@/share/kata-containers.img";
+              }
+              {
+                url = "file:///opt/edgeless/share/kata-kernel";
+                path = "/opt/edgeless/@@runtimeName@@/share/kata-kernel";
+              }
+              {
+                url = "file:///opt/edgeless/share/kata-initrd.zst";
+                path = "/opt/edgeless/@@runtimeName@@/share/kata-initrd.zst";
+              }
+              {
+                url = "file:///opt/edgeless/snp/bin/qemu-system-x86_64";
+                path = "/opt/edgeless/@@runtimeName@@/snp/bin/qemu-system-x86_64";
+                executable = true;
+              }
+              {
+                url = "file:///opt/edgeless/snp/share/OVMF.fd";
+                path = "/opt/edgeless/@@runtimeName@@/snp/share/OVMF.fd";
+              }
+              {
+                url = "file:///opt/edgeless/bin/containerd-shim-contrast-cc-v2";
+                path = "/opt/edgeless/@@runtimeName@@/bin/containerd-shim-contrast-cc-v2";
+                executable = true;
+              }
+              {
+                url = "file:///opt/edgeless/bin/kata-runtime";
+                path = "/opt/edgeless/@@runtimeName@@/bin/kata-runtime";
+                executable = true;
+              }
+              {
+                url = "file:///opt/edgeless/snp/share/qemu/kvmvapic.bin";
+                path = "/opt/edgeless/@@runtimeName@@/snp/share/qemu/kvmvapic.bin";
+              }
+              {
+                url = "file:///opt/edgeless/snp/share/qemu/linuxboot_dma.bin";
+                path = "/opt/edgeless/@@runtimeName@@/snp/share/qemu/linuxboot_dma.bin";
+              }
+              {
+                url = "file:///opt/edgeless/snp/share/qemu/efi-virtio.rom";
+                path = "/opt/edgeless/@@runtimeName@@/snp/share/qemu/efi-virtio.rom";
+              }
+              {
+                url = "file:///bin/nydus-overlayfs";
+                path = "/opt/edgeless/@@runtimeName@@/bin/nydus-overlayfs";
+                executable = true;
+              }
+            ]
+            ++ lib.optionals (!withGPU) [
+              {
+                url = "file:///opt/edgeless/tdx/share/OVMF.fd";
+                path = "/opt/edgeless/@@runtimeName@@/tdx/share/OVMF.fd";
+              }
+              {
+                url = "file:///opt/edgeless/tdx/bin/qemu-system-x86_64";
+                path = "/opt/edgeless/@@runtimeName@@/tdx/bin/qemu-system-x86_64";
+                executable = true;
+              }
+              {
+                url = "file:///opt/edgeless/tdx/share/qemu/kvmvapic.bin";
+                path = "/opt/edgeless/@@runtimeName@@/tdx/share/qemu/kvmvapic.bin";
+              }
+              {
+                url = "file:///opt/edgeless/tdx/share/qemu/linuxboot_dma.bin";
+                path = "/opt/edgeless/@@runtimeName@@/tdx/share/qemu/linuxboot_dma.bin";
+              }
+              {
+                url = "file:///opt/edgeless/tdx/share/qemu/efi-virtio.rom";
+                path = "/opt/edgeless/@@runtimeName@@/tdx/share/qemu/efi-virtio.rom";
+              }
+            ];
           inherit debugRuntime;
-          qemuExtraKernelParams = kata.kata-image.cmdline;
+          qemuExtraKernelParams = os-image.cmdline;
         };
         destination = "/config/contrast-node-install.json";
       }
@@ -120,15 +130,15 @@ let
   kata-container-img = ociLayerTar {
     files = [
       {
-        source = "${kata.kata-image.image}/${kata.kata-image.imageFileName}";
+        source = "${os-image.image}/${os-image.imageFileName}";
         destination = "/opt/edgeless/share/kata-containers.img";
       }
       {
-        source = "${kata.kata-image.kernel}/bzImage";
+        source = "${os-image.kernel}/bzImage";
         destination = "/opt/edgeless/share/kata-kernel";
       }
       {
-        source = "${kata.kata-image.initialRamdisk}/initrd";
+        source = "${os-image.initialRamdisk}/initrd";
         destination = "/opt/edgeless/share/kata-initrd.zst";
       }
     ];
@@ -216,16 +226,19 @@ let
     ];
   };
 
-  layers = [
-    installer-config
-    kata-container-img
-    ovmf-snp
-    ovmf-tdx
-    qemu-snp
-    qemu-tdx
-    kata-runtime
-    nydus
-  ];
+  layers =
+    [
+      installer-config
+      kata-container-img
+      kata-runtime
+      ovmf-snp
+      qemu-snp
+      nydus
+    ]
+    ++ lib.optionals (!withGPU) [
+      qemu-tdx
+      ovmf-tdx
+    ];
 
   manifest = ociImageManifest {
     layers = layers ++ [ node-installer ];
@@ -251,10 +264,14 @@ in
 ociImageLayout {
   manifests = [ manifest ];
   passthru = {
-    inherit debugRuntime;
+    inherit debugRuntime os-image;
     runtimeHash = hashDirs {
       dirs = layers; # Layers without node-installer, or we have a circular dependency!
       name = "runtime-hash-kata";
     };
+    gpu = kata.contrast-node-installer-image.override {
+      inherit debugRuntime;
+      withGPU = true;
+    };
   };
 }
diff --git a/packages/by-name/kata/snp-launch-digest/package.nix b/packages/by-name/kata/snp-launch-digest/package.nix
@@ -4,30 +4,31 @@
 {
   lib,
   stdenvNoCC,
-  kata,
   OVMF-SNP,
   python3Packages,
+  kata,
 
   debug ? false,
+  os-image ? kata.kata-image,
 }:
 
 let
   ovmf-snp = "${OVMF-SNP}/FV/OVMF.fd";
-  kernel = "${kata.kata-image}/bzImage";
-  initrd = "${kata.kata-image}/initrd";
+  kernel = "${os-image}/bzImage";
+  initrd = "${os-image}/initrd";
 
   # Kata uses a base command line and then appends the command line from the kata config (i.e. also our node-installer config).
   # Thus, we need to perform the same steps when calculating the digest.
   baseCmdline = if debug then kata.kata-runtime.cmdline.debug else kata.kata-runtime.cmdline.default;
   cmdline = lib.strings.concatStringsSep " " [
     baseCmdline
-    kata.kata-image.cmdline
+    os-image.cmdline
   ];
 in
 
 stdenvNoCC.mkDerivation {
   name = "snp-launch-digest${lib.optionalString debug "-debug"}";
-  inherit (kata.kata-image) version;
+  inherit (os-image) version;
 
   dontUnpack = true;
 

diff --git a/packages/containers.nix b/packages/containers.nix
@@ -188,5 +188,9 @@ containers
   push-node-installer-kata =
     pushOCIDir "push-node-installer-kata" pkgs.kata.contrast-node-installer-image
       "v${pkgs.contrast.version}";
+  push-node-installer-kata-gpu = pushOCIDir "push-node-installer-kata-gpu" (
+    pkgs.kata.contrast-node-installer-image.override
+    { withGPU = true; }
+  ) "v${pkgs.contrast.version}";
 }
 // (lib.concatMapAttrs (name: container: { "push-${name}" = pushContainer container; }) containers)
diff --git a/packages/nixos/kata.nix b/packages/nixos/kata.nix
@@ -78,7 +78,13 @@ in
     };
 
     # Not used directly, but required for kernel-specific driver builds.
-    boot.kernelPackages = pkgs.recurseIntoAttrs (pkgs.linuxPackagesFor pkgs.kata-kernel-uvm);
+    boot.kernelPackages = pkgs.recurseIntoAttrs (
+      pkgs.linuxPackagesFor (
+        pkgs.kata-kernel-uvm.override {
+          withGPU = config.contrast.gpu.enable;
+        }
+      )
+    );
 
     boot.initrd = {
       # Don't require TPM2 support. (additional modules)