From 0edb5be8f26c41238630e6e1def0e82d0ed7dd97 Mon Sep 17 00:00:00 2001 From: Magnus Kulke Date: Sun, 15 Dec 2024 20:59:54 +0100 Subject: [PATCH] podvm: add scratch-space for /run/container This adds the configuration for an encrypted scratch space on an mkosi image. At bootup a /dev/sda4 partition will be created and encrypted with LUKS using an ephemeral key. The partition will use the space available on the image volume. By default the qcow2 image has 100mb allocated for this space. This amount of space will only work for very small images, hence we do not mount the scratch space to `/run/kata-container` by default. If the kata-agent service units encounters a `/run/peerpod/mount-scratch` file it will mount the encrypted partition `/dev/sda4` to `/run/kata-containers`. This file is provisioned by `process-user-data`, configured by the CAA daemonset. A new CAA parameter has been introduced that allows to specify the disk size. If we have a disk size that is larger than 0 a cloud provider can attempt to consider this size to create space for an encrypted scratch partition that can be mounted to /run/container. Signed-off-by: Magnus Kulke --- .../cmd/cloud-api-adaptor/main.go | 1 + src/cloud-api-adaptor/entrypoint.sh | 1 + .../pkg/adaptor/cloud/cloud.go | 20 +- src/cloud-api-adaptor/pkg/paths/paths.go | 1 + .../pkg/userdata/provision.go | 2 +- src/cloud-api-adaptor/pkg/util/cloud.go | 31 ++- src/cloud-api-adaptor/pkg/util/cloud_test.go | 3 +- src/cloud-api-adaptor/podvm-mkosi/Makefile | 2 + .../system/mkosi.conf.d/fedora.conf | 3 +- .../mkosi.skeleton-rootfs/etc/crypttab | 1 + .../etc/neofetch/coco.ascii | 0 .../etc/neofetch/config.conf | 0 .../etc/profile.d/10-alias.sh | 0 .../etc/profile.d/20-ssh-banner.sh | 0 .../10-override.conf | 0 .../usr/lib/systemd/system/gen-issue.service | 0 .../kata-agent.service.d/10-override.conf | 2 + .../10-override.conf | 0 .../usr/lib/repart.d/30-scratch.conf | 5 + src/cloud-providers/aws/provider.go | 4 +- src/cloud-providers/azure/provider.go | 78 +++--- src/cloud-providers/ibmcloud/provider.go | 3 +- src/cloud-providers/types.go | 20 +- src/cloud-providers/util.go | 30 +- src/cloud-providers/util_test.go | 260 ++++++++++++------ 25 files changed, 300 insertions(+), 167 deletions(-) create mode 100644 src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/crypttab rename src/cloud-api-adaptor/podvm-mkosi/{mkosi.skeleton => mkosi.skeleton-rootfs}/etc/neofetch/coco.ascii (100%) rename src/cloud-api-adaptor/podvm-mkosi/{mkosi.skeleton => mkosi.skeleton-rootfs}/etc/neofetch/config.conf (100%) rename src/cloud-api-adaptor/podvm-mkosi/{mkosi.skeleton => mkosi.skeleton-rootfs}/etc/profile.d/10-alias.sh (100%) rename src/cloud-api-adaptor/podvm-mkosi/{mkosi.skeleton => mkosi.skeleton-rootfs}/etc/profile.d/20-ssh-banner.sh (100%) rename src/cloud-api-adaptor/podvm-mkosi/{mkosi.skeleton => mkosi.skeleton-rootfs}/usr/lib/systemd/system/afterburn-checkin.service.d/10-override.conf (100%) rename src/cloud-api-adaptor/podvm-mkosi/{mkosi.skeleton => mkosi.skeleton-rootfs}/usr/lib/systemd/system/gen-issue.service (100%) create mode 100644 src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/kata-agent.service.d/10-override.conf rename src/cloud-api-adaptor/podvm-mkosi/{mkosi.skeleton => mkosi.skeleton-rootfs}/usr/lib/systemd/system/process-user-data.service.d/10-override.conf (100%) create mode 100644 src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/repart.d/30-scratch.conf diff --git a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go index 5f9e176354..bdd4a70575 100644 --- a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go +++ b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go @@ -125,6 +125,7 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { flags.StringVar(&cfg.serverConfig.Initdata, "initdata", "", "Default initdata for all Pods") flags.BoolVar(&cfg.serverConfig.EnableCloudConfigVerify, "cloud-config-verify", false, "Enable cloud config verify - should use it for production") flags.IntVar(&cfg.serverConfig.PeerPodsLimitPerNode, "peerpods-limit-per-node", 10, "peer pods limit per node (default=10)") + flags.Uint64Var(&cfg.serverConfig.RootVolumeSize, "root-volume-size", 0, "Root volume size in GB. Default is 0, which implies the default image disk size") cloud.ParseCmd(flags) }) diff --git a/src/cloud-api-adaptor/entrypoint.sh b/src/cloud-api-adaptor/entrypoint.sh index 6f32c4917e..7b176c0045 100755 --- a/src/cloud-api-adaptor/entrypoint.sh +++ b/src/cloud-api-adaptor/entrypoint.sh @@ -77,6 +77,7 @@ azure() { [[ "${TAGS}" ]] && optionals+="-tags ${TAGS} " # Custom tags applied to pod vm [[ "${ENABLE_SECURE_BOOT}" == "true" ]] && optionals+="-enable-secure-boot " [[ "${USE_PUBLIC_IP}" == "true" ]] && optionals+="-use-public-ip " + [[ "${ROOT_VOLUME_SIZE}" ]] && optionals+="-root-volume-size ${ROOT_VOLUME_SIZE} " # OS disk size in GB set -x exec cloud-api-adaptor azure \ diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go index 8a88eb9cd8..5b7aadbfaf 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go @@ -63,6 +63,7 @@ type ServerConfig struct { SecureCommsPpOutbounds string SecureCommsKbsAddress string PeerPodsLimitPerNode int + RootVolumeSize uint64 } var logger = log.New(log.Writer(), "[adaptor/cloud] ", log.LstdFlags|log.Lmsgprefix) @@ -221,7 +222,12 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r instanceType := util.GetInstanceTypeFromAnnotation(req.Annotations) // Get Pod VM cpu and memory from annotations - vcpus, memory, gpus := util.GetPodvmResourcesFromAnnotation(req.Annotations) + resources := util.GetPodVMResourcesFromAnnotation(req.Annotations) + + // Set caa-wide root volume size to resources if the storage has not been configured yet + if resources.Storage == 0 && s.serverConfig.RootVolumeSize > 0 { + resources.Storage = int64(s.serverConfig.RootVolumeSize) + } // Get Pod VM image from annotations image := util.GetImageFromAnnotation(req.Annotations) @@ -229,9 +235,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r // Pod VM spec vmSpec := provider.InstanceTypeSpec{ InstanceType: instanceType, - VCPUs: vcpus, - Memory: memory, - GPUs: gpus, + Resources: resources, Image: image, } @@ -320,6 +324,14 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r }, } + if s.serverConfig.RootVolumeSize > 0 { + // Write an empty file to indicate that we want to use available space as sandbox storage + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ + Path: UseScratchPath, + Content: "", + }) + } + if authJSON != nil { if len(authJSON) > cloudinit.DefaultAuthfileLimit { logger.Printf("Credentials file is too large to be included in cloud-config") diff --git a/src/cloud-api-adaptor/pkg/paths/paths.go b/src/cloud-api-adaptor/pkg/paths/paths.go index 8b081b1b75..a84ebdfdb8 100644 --- a/src/cloud-api-adaptor/pkg/paths/paths.go +++ b/src/cloud-api-adaptor/pkg/paths/paths.go @@ -8,4 +8,5 @@ const ( AgentCfgPath = "/run/peerpod/agent-config.toml" ForwarderCfgPath = "/run/peerpod/daemon.json" UserDataPath = "/media/cidata/user-data" + UseScratchPath = "/run/peerpod/mount-scratch" ) diff --git a/src/cloud-api-adaptor/pkg/userdata/provision.go b/src/cloud-api-adaptor/pkg/userdata/provision.go index 2f780a00bb..c32285b8f0 100644 --- a/src/cloud-api-adaptor/pkg/userdata/provision.go +++ b/src/cloud-api-adaptor/pkg/userdata/provision.go @@ -37,7 +37,7 @@ const ( ) var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix) -var WriteFilesList = []string{AACfgPath, CDHCfgPath, ForwarderCfgPath, AuthFilePath, InitDataPath} +var WriteFilesList = []string{AACfgPath, CDHCfgPath, ForwarderCfgPath, AuthFilePath, InitDataPath, UseScratchPath} var InitdDataFilesList = []string{AACfgPath, CDHCfgPath, PolicyPath} type Config struct { diff --git a/src/cloud-api-adaptor/pkg/util/cloud.go b/src/cloud-api-adaptor/pkg/util/cloud.go index 9c0dfda811..859eec5feb 100644 --- a/src/cloud-api-adaptor/pkg/util/cloud.go +++ b/src/cloud-api-adaptor/pkg/util/cloud.go @@ -5,6 +5,7 @@ import ( "strconv" "strings" + provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers" cri "github.com/containerd/containerd/pkg/cri/annotations" hypannotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" ) @@ -44,48 +45,48 @@ func GetImageFromAnnotation(annotations map[string]string) string { } // Method to get vCPU, memory and gpus from annotations -func GetPodvmResourcesFromAnnotation(annotations map[string]string) (int64, int64, int64) { - - var vcpuInt, memoryInt, gpuInt int64 +func GetPodVMResourcesFromAnnotation(annotations map[string]string) provider.PodVMResources { + var vcpus, gpus, memory int64 var err error vcpu, ok := annotations[hypannotations.DefaultVCPUs] if ok { - vcpuInt, err = strconv.ParseInt(vcpu, 10, 64) + vcpus, err = strconv.ParseInt(vcpu, 10, 64) if err != nil { fmt.Printf("Error converting vcpu to int64. Defaulting to 0: %v\n", err) - vcpuInt = 0 + vcpus = 0 } } else { - vcpuInt = 0 + vcpus = 0 } - memory, ok := annotations[hypannotations.DefaultMemory] + mem, ok := annotations[hypannotations.DefaultMemory] if ok { // Use strconv.ParseInt to convert string to int64 - memoryInt, err = strconv.ParseInt(memory, 10, 64) + memory, err = strconv.ParseInt(mem, 10, 64) if err != nil { fmt.Printf("Error converting memory to int64. Defaulting to 0: %v\n", err) - memoryInt = 0 + memory = 0 } } else { - memoryInt = 0 + memory = 0 } gpu, ok := annotations[hypannotations.DefaultGPUs] if ok { - gpuInt, err = strconv.ParseInt(gpu, 10, 64) + gpus, err = strconv.ParseInt(gpu, 10, 64) if err != nil { fmt.Printf("Error converting gpu to int64. Defaulting to 0: %v\n", err) - gpuInt = 0 + gpus = 0 } } else { - gpuInt = 0 + gpus = 0 } - // Return vCPU, memory and GPU - return vcpuInt, memoryInt, gpuInt + storage := int64(0) + + return provider.PodVMResources{VCPUs: vcpus, Memory: memory, GPUs: gpus, Storage: storage} } // Method to get initdata from annotation diff --git a/src/cloud-api-adaptor/pkg/util/cloud_test.go b/src/cloud-api-adaptor/pkg/util/cloud_test.go index 3e1a2c8594..dc152a421e 100644 --- a/src/cloud-api-adaptor/pkg/util/cloud_test.go +++ b/src/cloud-api-adaptor/pkg/util/cloud_test.go @@ -98,7 +98,8 @@ func TestGetPodvmResourcesFromAnnotation(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, got1, got2 := GetPodvmResourcesFromAnnotation(tt.args.annotations) + r := GetPodVMResourcesFromAnnotation(tt.args.annotations) + got, got1, got2 := r.VCPUs, r.Memory, r.GPUs if got != tt.want { t.Errorf("GetPodvmResourcesFromAnnotation() got = %v, want %v", got, tt.want) } diff --git a/src/cloud-api-adaptor/podvm-mkosi/Makefile b/src/cloud-api-adaptor/podvm-mkosi/Makefile index e9edf6a533..b5d4e95c98 100644 --- a/src/cloud-api-adaptor/podvm-mkosi/Makefile +++ b/src/cloud-api-adaptor/podvm-mkosi/Makefile @@ -88,6 +88,7 @@ else --allow security.insecure \ . qemu-img convert -f raw -O qcow2 build/system.raw build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2 + qemu-img resize build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2 +50M endif PHONY: image-debug @@ -112,6 +113,7 @@ else --allow security.insecure \ . qemu-img convert -f raw -O qcow2 build/system.raw build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2 + qemu-img resize build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2 +50M endif PHONY: image-container diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.presets/system/mkosi.conf.d/fedora.conf b/src/cloud-api-adaptor/podvm-mkosi/mkosi.presets/system/mkosi.conf.d/fedora.conf index 141470e7ae..5abd00514f 100644 --- a/src/cloud-api-adaptor/podvm-mkosi/mkosi.presets/system/mkosi.conf.d/fedora.conf +++ b/src/cloud-api-adaptor/podvm-mkosi/mkosi.presets/system/mkosi.conf.d/fedora.conf @@ -7,7 +7,7 @@ Release=39 [Content] CleanPackageMetadata=true -SkeletonTrees=../../resources/binaries-tree +SkeletonTrees=../../mkosi.skeleton-rootfs,../../resources/binaries-tree, Packages= kernel kernel-core @@ -23,6 +23,7 @@ Packages= iptables afterburn neofetch + e2fsprogs RemoveFiles=/etc/issue RemoveFiles=/etc/issue.net diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/crypttab b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/crypttab new file mode 100644 index 0000000000..c91cea6d4e --- /dev/null +++ b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/crypttab @@ -0,0 +1 @@ +scratch /dev/disk/by-label/scratch - try-empty-password diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/etc/neofetch/coco.ascii b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/neofetch/coco.ascii similarity index 100% rename from src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/etc/neofetch/coco.ascii rename to src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/neofetch/coco.ascii diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/etc/neofetch/config.conf b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/neofetch/config.conf similarity index 100% rename from src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/etc/neofetch/config.conf rename to src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/neofetch/config.conf diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/etc/profile.d/10-alias.sh b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/profile.d/10-alias.sh similarity index 100% rename from src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/etc/profile.d/10-alias.sh rename to src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/profile.d/10-alias.sh diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/etc/profile.d/20-ssh-banner.sh b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/profile.d/20-ssh-banner.sh similarity index 100% rename from src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/etc/profile.d/20-ssh-banner.sh rename to src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/profile.d/20-ssh-banner.sh diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/systemd/system/afterburn-checkin.service.d/10-override.conf b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/afterburn-checkin.service.d/10-override.conf similarity index 100% rename from src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/systemd/system/afterburn-checkin.service.d/10-override.conf rename to src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/afterburn-checkin.service.d/10-override.conf diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/systemd/system/gen-issue.service b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/gen-issue.service similarity index 100% rename from src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/systemd/system/gen-issue.service rename to src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/gen-issue.service diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/kata-agent.service.d/10-override.conf b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/kata-agent.service.d/10-override.conf new file mode 100644 index 0000000000..a20832667e --- /dev/null +++ b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/kata-agent.service.d/10-override.conf @@ -0,0 +1,2 @@ +[Service] +ExecStartPre=sh -c '[[ ! -f /run/peerpod/mount-scratch ]] || mount /dev/mapper/scratch /run/kata-containers' diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/systemd/system/process-user-data.service.d/10-override.conf b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/process-user-data.service.d/10-override.conf similarity index 100% rename from src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/systemd/system/process-user-data.service.d/10-override.conf rename to src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/process-user-data.service.d/10-override.conf diff --git a/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/repart.d/30-scratch.conf b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/repart.d/30-scratch.conf new file mode 100644 index 0000000000..cabcc9ef5c --- /dev/null +++ b/src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/repart.d/30-scratch.conf @@ -0,0 +1,5 @@ +[Partition] +Type=linux-generic +Label=scratch +Encrypt=key-file +Format=ext4 diff --git a/src/cloud-providers/aws/provider.go b/src/cloud-providers/aws/provider.go index d47ba7f743..b473001441 100644 --- a/src/cloud-providers/aws/provider.go +++ b/src/cloud-providers/aws/provider.go @@ -370,8 +370,10 @@ func (p *awsProvider) updateInstanceTypeSpecList() error { if err != nil { return err } + resources := provider.NewPodVMResources(vcpus, memory) + resources.GPUs = gpuCount instanceTypeSpecList = append(instanceTypeSpecList, - provider.InstanceTypeSpec{InstanceType: instanceType, VCPUs: vcpus, Memory: memory, GPUs: gpuCount}) + provider.InstanceTypeSpec{InstanceType: instanceType, Resources: resources}) } // Sort the instanceTypeSpecList and update the serviceConfig diff --git a/src/cloud-providers/azure/provider.go b/src/cloud-providers/azure/provider.go index 4e4488a98d..50c2536ad1 100644 --- a/src/cloud-providers/azure/provider.go +++ b/src/cloud-providers/azure/provider.go @@ -11,7 +11,6 @@ import ( "log" "net/netip" "os" - "path/filepath" "regexp" "strings" @@ -35,24 +34,44 @@ const ( type azureProvider struct { azureClient azcore.TokenCredential serviceConfig *Config + sshKey armcompute.SSHPublicKey } func NewProvider(config *Config) (provider.Provider, error) { logger.Printf("azure config %+v", config.Redact()) - // Clean the config.SSHKeyPath to avoid bad paths - config.SSHKeyPath = filepath.Clean(config.SSHKeyPath) - azureClient, err := NewAzureClient(*config) if err != nil { logger.Printf("creating azure client: %v", err) return nil, err } + // The podvm disk doesn't support sshd logins. keys can be baked + // into a debug-image. The ARM api mandates a pubkey, though. + sshPublicKeyPath := os.ExpandEnv(config.SSHKeyPath) + var pubkeyBytes []byte + if _, err := os.Stat(sshPublicKeyPath); err == nil { + pubkeyBytes, err = os.ReadFile(sshPublicKeyPath) + if err != nil { + err = fmt.Errorf("reading ssh public key file: %w", err) + logger.Printf("%v", err) + return nil, err + } + } else { + err = fmt.Errorf("ssh public key: %w", err) + logger.Printf("%v", err) + return nil, err + } + dummySSHKey := armcompute.SSHPublicKey{ + Path: to.Ptr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", config.SSHUserName)), + KeyData: to.Ptr(string(pubkeyBytes)), + } + provider := &azureProvider{ azureClient: azureClient, serviceConfig: config, + sshKey: dummySSHKey, } if err = provider.updateInstanceSizeSpecList(); err != nil { @@ -218,28 +237,12 @@ func (p *azureProvider) CreateInstance(ctx context.Context, podName, sandboxID s diskName := fmt.Sprintf("%s-disk", instanceName) nicName := fmt.Sprintf("%s-net", instanceName) - // require ssh key for authentication on linux - sshPublicKeyPath := os.ExpandEnv(p.serviceConfig.SSHKeyPath) - var sshBytes []byte - if _, err := os.Stat(sshPublicKeyPath); err == nil { - sshBytes, err = os.ReadFile(sshPublicKeyPath) - if err != nil { - err = fmt.Errorf("reading ssh public key file: %w", err) - logger.Printf("%v", err) - return nil, err - } - } else { - err = fmt.Errorf("ssh public key: %w", err) - logger.Printf("%v", err) - return nil, err - } - if spec.Image != "" { logger.Printf("Choosing %s from annotation as the Azure Image for the PodVM image", spec.Image) p.serviceConfig.ImageId = spec.Image } - vmParameters, err := p.getVMParameters(instanceSize, diskName, cloudConfigData, sshBytes, instanceName, nicName) + vmParameters, err := p.getVMParameters(instanceSize, diskName, cloudConfigData, instanceName, nicName, spec.Resources.Storage) if err != nil { return nil, err } @@ -350,7 +353,10 @@ func (p *azureProvider) updateInstanceSizeSpecList() error { } for _, vmSize := range nextResult.VirtualMachineSizeListResult.Value { if util.Contains(instanceSizes, *vmSize.Name) { - instanceSizeSpecList = append(instanceSizeSpecList, provider.InstanceTypeSpec{InstanceType: *vmSize.Name, VCPUs: int64(*vmSize.NumberOfCores), Memory: int64(*vmSize.MemoryInMB)}) + vcpus, memory := int64(*vmSize.NumberOfCores), int64(*vmSize.MemoryInMB) + resources := provider.NewPodVMResources(vcpus, memory) + instanceSizeSpec := provider.InstanceTypeSpec{InstanceType: *vmSize.Name, Resources: resources} + instanceSizeSpecList = append(instanceSizeSpecList, instanceSizeSpec) } } } @@ -371,7 +377,7 @@ func (p *azureProvider) getResourceTags() map[string]*string { return tags } -func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig string, sshBytes []byte, instanceName, nicName string) (*armcompute.VirtualMachine, error) { +func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig, instanceName, nicName string, diskSize int64) (*armcompute.VirtualMachine, error) { userDataB64 := base64.StdEncoding.EncodeToString([]byte(cloudConfig)) // Azure limits the base64 encrypted userData to 64KB. @@ -416,6 +422,18 @@ func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig stri networkConfig := p.buildNetworkConfig(nicName) + osDisk := armcompute.OSDisk{ + Name: to.Ptr(diskName), + CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesFromImage), + Caching: to.Ptr(armcompute.CachingTypesReadWrite), + DeleteOption: to.Ptr(armcompute.DiskDeleteOptionTypesDelete), + ManagedDisk: managedDiskParams, + } + + if diskSize > 0 { + osDisk.DiskSizeGB = to.Ptr(int32(diskSize)) + } + vmParameters := armcompute.VirtualMachine{ Location: to.Ptr(p.serviceConfig.Region), Properties: &armcompute.VirtualMachineProperties{ @@ -424,25 +442,15 @@ func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig stri }, StorageProfile: &armcompute.StorageProfile{ ImageReference: imgRef, - OSDisk: &armcompute.OSDisk{ - Name: to.Ptr(diskName), - CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesFromImage), - Caching: to.Ptr(armcompute.CachingTypesReadWrite), - DeleteOption: to.Ptr(armcompute.DiskDeleteOptionTypesDelete), - ManagedDisk: managedDiskParams, - }, + OSDisk: &osDisk, }, OSProfile: &armcompute.OSProfile{ AdminUsername: to.Ptr(p.serviceConfig.SSHUserName), ComputerName: to.Ptr(instanceName), LinuxConfiguration: &armcompute.LinuxConfiguration{ DisablePasswordAuthentication: to.Ptr(true), - //TBD: replace with a suitable mechanism to use precreated SSH key SSH: &armcompute.SSHConfiguration{ - PublicKeys: []*armcompute.SSHPublicKey{{ - Path: to.Ptr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", p.serviceConfig.SSHUserName)), - KeyData: to.Ptr(string(sshBytes)), - }}, + PublicKeys: []*armcompute.SSHPublicKey{&p.sshKey}, }, }, }, diff --git a/src/cloud-providers/ibmcloud/provider.go b/src/cloud-providers/ibmcloud/provider.go index 1be306405d..0dcee2d1d0 100644 --- a/src/cloud-providers/ibmcloud/provider.go +++ b/src/cloud-providers/ibmcloud/provider.go @@ -332,7 +332,8 @@ func (p *ibmcloudVPCProvider) updateInstanceProfileSpecList() error { if err != nil { return err } - instanceProfileSpecList = append(instanceProfileSpecList, provider.InstanceTypeSpec{InstanceType: profileType, VCPUs: vcpus, Memory: memory, Arch: arch}) + resources := provider.NewPodVMResources(vcpus, memory) + instanceProfileSpecList = append(instanceProfileSpecList, provider.InstanceTypeSpec{InstanceType: profileType, Resources: resources, Arch: arch}) } // Sort the instanceProfileSpecList and update the serviceConfig diff --git a/src/cloud-providers/types.go b/src/cloud-providers/types.go index 82ea5b0c7e..d1a093225d 100644 --- a/src/cloud-providers/types.go +++ b/src/cloud-providers/types.go @@ -60,11 +60,25 @@ type Instance struct { IPs []netip.Addr } +type PodVMResources struct { + VCPUs int64 + Memory int64 + GPUs int64 + Storage int64 +} + +func NewPodVMResources(vcpus, memory int64) PodVMResources { + return PodVMResources{ + VCPUs: vcpus, + Memory: memory, + GPUs: 0, + Storage: 0, + } +} + type InstanceTypeSpec struct { InstanceType string - VCPUs int64 - Memory int64 + Resources PodVMResources Arch string - GPUs int64 Image string } diff --git a/src/cloud-providers/util.go b/src/cloud-providers/util.go index e8b3f63098..40268dd1d8 100644 --- a/src/cloud-providers/util.go +++ b/src/cloud-providers/util.go @@ -44,16 +44,17 @@ func VerifyCloudInstanceType(instanceType string, validInstanceTypes []string, d // Method to sort InstanceTypeSpec into ascending order based on gpu, then memory, followed by cpu func SortInstanceTypesOnResources(instanceTypeSpecList []InstanceTypeSpec) []InstanceTypeSpec { sort.Slice(instanceTypeSpecList, func(i, j int) bool { + resI, resJ := instanceTypeSpecList[i].Resources, instanceTypeSpecList[j].Resources // First, sort by GPU count - if instanceTypeSpecList[i].GPUs != instanceTypeSpecList[j].GPUs { - return instanceTypeSpecList[i].GPUs < instanceTypeSpecList[j].GPUs + if resI.GPUs != resJ.GPUs { + return resI.GPUs < resJ.GPUs } // If GPU count is the same, sort by memory - if instanceTypeSpecList[i].Memory != instanceTypeSpecList[j].Memory { - return instanceTypeSpecList[i].Memory < instanceTypeSpecList[j].Memory + if resI.Memory != resJ.Memory { + return resI.Memory < resJ.Memory } // If memory is the same, sort by vCPUs - return instanceTypeSpecList[i].VCPUs < instanceTypeSpecList[j].VCPUs + return resI.VCPUs < resJ.VCPUs }) return instanceTypeSpecList @@ -64,16 +65,17 @@ func SelectInstanceTypeToUse(spec InstanceTypeSpec, specList []InstanceTypeSpec, var instanceType string var err error + gpus, vcpus, memory := spec.Resources.GPUs, spec.Resources.VCPUs, spec.Resources.Memory // GPU gets the highest priority - if spec.GPUs > 0 { - instanceType, err = GetBestFitInstanceTypeWithGPU(specList, spec.GPUs, spec.VCPUs, spec.Memory) + if gpus > 0 { + instanceType, err = GetBestFitInstanceTypeWithGPU(specList, gpus, vcpus, memory) if err != nil { return "", fmt.Errorf("failed to get instance type based on GPU, vCPU, and memory annotations: %w", err) } logger.Printf("Instance type selected by the cloud provider based on GPU annotation: %s", instanceType) - } else if spec.VCPUs != 0 && spec.Memory != 0 { + } else if vcpus != 0 && memory != 0 { // If no GPU is required, fall back to vCPU and memory selection - instanceType, err = GetBestFitInstanceType(specList, spec.VCPUs, spec.Memory) + instanceType, err = GetBestFitInstanceType(specList, vcpus, memory) if err != nil { return "", fmt.Errorf("failed to get instance type based on vCPU and memory annotations: %w", err) } @@ -104,7 +106,8 @@ func GetBestFitInstanceType(sortedInstanceTypeSpecList []InstanceTypeSpec, vcpus // Use sort.Search to find the index of the first element in the sortedMachineTypeList slice // that is greater than or equal to the given memory and vcpus index := sort.Search(len(sortedInstanceTypeSpecList), func(i int) bool { - return sortedInstanceTypeSpecList[i].Memory >= memory && sortedInstanceTypeSpecList[i].VCPUs >= vcpus + res := sortedInstanceTypeSpecList[i].Resources + return res.Memory >= memory && res.VCPUs >= vcpus }) // If binary search fails to find a match, return error @@ -121,7 +124,7 @@ func GetBestFitInstanceType(sortedInstanceTypeSpecList []InstanceTypeSpec, vcpus func FilterOutGPUInstances(instanceTypeSpecList []InstanceTypeSpec) []InstanceTypeSpec { var filteredList []InstanceTypeSpec for _, spec := range instanceTypeSpecList { - if spec.GPUs == 0 { + if spec.Resources.GPUs == 0 { filteredList = append(filteredList, spec) } } @@ -132,9 +135,8 @@ func FilterOutGPUInstances(instanceTypeSpecList []InstanceTypeSpec) []InstanceTy // TBD: Incorporate GPU model based selection as well func GetBestFitInstanceTypeWithGPU(sortedInstanceTypeSpecList []InstanceTypeSpec, gpus, vcpus, memory int64) (string, error) { index := sort.Search(len(sortedInstanceTypeSpecList), func(i int) bool { - return sortedInstanceTypeSpecList[i].GPUs >= gpus && - sortedInstanceTypeSpecList[i].VCPUs >= vcpus && - sortedInstanceTypeSpecList[i].Memory >= memory + res := sortedInstanceTypeSpecList[i].Resources + return res.GPUs >= gpus && res.VCPUs >= vcpus && res.Memory >= memory }) if index == len(sortedInstanceTypeSpecList) { diff --git a/src/cloud-providers/util_test.go b/src/cloud-providers/util_test.go index 83aa0845a7..c1525f3339 100644 --- a/src/cloud-providers/util_test.go +++ b/src/cloud-providers/util_test.go @@ -109,36 +109,48 @@ func TestSortInstanceTypesOnResources(t *testing.T) { instanceTypeSpecList: []InstanceTypeSpec{ { InstanceType: "t2.small", - VCPUs: 2, - Memory: 6, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + }, }, { InstanceType: "t2.medium", - VCPUs: 4, - Memory: 8, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + }, }, { InstanceType: "t2.large", - VCPUs: 8, - Memory: 16, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + }, }, }, }, want: []InstanceTypeSpec{ { InstanceType: "t2.small", - VCPUs: 2, - Memory: 6, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + }, }, { InstanceType: "t2.medium", - VCPUs: 4, - Memory: 8, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + }, }, { InstanceType: "t2.large", - VCPUs: 8, - Memory: 16, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + }, }, }, }, @@ -149,36 +161,48 @@ func TestSortInstanceTypesOnResources(t *testing.T) { instanceTypeSpecList: []InstanceTypeSpec{ { InstanceType: "t2.small", - VCPUs: 2, - Memory: 6, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + }, }, { InstanceType: "t2.large", - VCPUs: 8, - Memory: 16, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + }, }, { InstanceType: "t2.medium", - VCPUs: 4, - Memory: 8, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + }, }, }, }, want: []InstanceTypeSpec{ { InstanceType: "t2.small", - VCPUs: 2, - Memory: 6, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + }, }, { InstanceType: "t2.medium", - VCPUs: 4, - Memory: 8, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + }, }, { InstanceType: "t2.large", - VCPUs: 8, - Memory: 16, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + }, }, }, }, @@ -189,36 +213,48 @@ func TestSortInstanceTypesOnResources(t *testing.T) { instanceTypeSpecList: []InstanceTypeSpec{ { InstanceType: "t2.medium", - VCPUs: 4, - Memory: 8, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + }, }, { InstanceType: "t2.small", - VCPUs: 2, - Memory: 6, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + }, }, { InstanceType: "t2.large", - VCPUs: 8, - Memory: 16, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + }, }, }, }, want: []InstanceTypeSpec{ { InstanceType: "t2.small", - VCPUs: 2, - Memory: 6, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + }, }, { InstanceType: "t2.medium", - VCPUs: 4, - Memory: 8, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + }, }, { InstanceType: "t2.large", - VCPUs: 8, - Memory: 16, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + }, }, }, }, @@ -230,42 +266,54 @@ func TestSortInstanceTypesOnResources(t *testing.T) { instanceTypeSpecList: []InstanceTypeSpec{ { InstanceType: "p2.medium", - VCPUs: 4, - Memory: 8, - GPUs: 2, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + GPUs: 2, + }, }, { InstanceType: "p2.small", - VCPUs: 2, - Memory: 6, - GPUs: 1, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + GPUs: 1, + }, }, { InstanceType: "p2.large", - VCPUs: 8, - Memory: 16, - GPUs: 4, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + GPUs: 4, + }, }, }, }, want: []InstanceTypeSpec{ { InstanceType: "p2.small", - VCPUs: 2, - Memory: 6, - GPUs: 1, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + GPUs: 1, + }, }, { InstanceType: "p2.medium", - VCPUs: 4, - Memory: 8, - GPUs: 2, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + GPUs: 2, + }, }, { InstanceType: "p2.large", - VCPUs: 8, - Memory: 16, - GPUs: 4, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + GPUs: 4, + }, }, }, }, @@ -302,18 +350,24 @@ func TestGetBestFitInstanceType(t *testing.T) { sortedInstanceTypeSpecList: []InstanceTypeSpec{ { InstanceType: "t2.small", - VCPUs: 2, - Memory: 6, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + }, }, { InstanceType: "t2.medium", - VCPUs: 4, - Memory: 8, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + }, }, { InstanceType: "t2.large", - VCPUs: 8, - Memory: 16, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + }, }, }, vcpus: 2, @@ -329,18 +383,24 @@ func TestGetBestFitInstanceType(t *testing.T) { sortedInstanceTypeSpecList: []InstanceTypeSpec{ { InstanceType: "t2.small", - VCPUs: 2, - Memory: 6, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + }, }, { InstanceType: "t2.medium", - VCPUs: 4, - Memory: 8, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + }, }, { InstanceType: "t2.large", - VCPUs: 8, - Memory: 16, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + }, }, }, vcpus: 4, @@ -356,18 +416,24 @@ func TestGetBestFitInstanceType(t *testing.T) { sortedInstanceTypeSpecList: []InstanceTypeSpec{ { InstanceType: "t2.small", - VCPUs: 2, - Memory: 6, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + }, }, { InstanceType: "t2.medium", - VCPUs: 4, - Memory: 8, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + }, }, { InstanceType: "t2.large", - VCPUs: 8, - Memory: 16, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + }, }, }, vcpus: 4, @@ -383,19 +449,25 @@ func TestGetBestFitInstanceType(t *testing.T) { sortedInstanceTypeSpecList: []InstanceTypeSpec{ { InstanceType: "t2.small", - VCPUs: 2, - Memory: 6, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + }, }, { InstanceType: "p2.medium", - VCPUs: 4, - Memory: 8, - GPUs: 2, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + GPUs: 2, + }, }, { InstanceType: "t2.large", - VCPUs: 8, - Memory: 16, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + }, }, }, vcpus: 4, @@ -412,18 +484,24 @@ func TestGetBestFitInstanceType(t *testing.T) { sortedInstanceTypeSpecList: []InstanceTypeSpec{ { InstanceType: "t2.small", - VCPUs: 2, - Memory: 6, + Resources: PodVMResources{ + VCPUs: 2, + Memory: 6, + }, }, { InstanceType: "t2.medium", - VCPUs: 4, - Memory: 8, + Resources: PodVMResources{ + VCPUs: 4, + Memory: 8, + }, }, { InstanceType: "t2.large", - VCPUs: 8, - Memory: 16, + Resources: PodVMResources{ + VCPUs: 8, + Memory: 16, + }, }, }, vcpus: 4, @@ -547,8 +625,8 @@ func TestGetBestFitInstanceTypeWithGPU(t *testing.T) { { name: "exact match", specList: []InstanceTypeSpec{ - {InstanceType: "small-gpu", GPUs: 1, VCPUs: 2, Memory: 4096}, - {InstanceType: "medium-gpu", GPUs: 2, VCPUs: 4, Memory: 8192}, + {InstanceType: "small-gpu", Resources: PodVMResources{GPUs: 1, VCPUs: 2, Memory: 4096}}, + {InstanceType: "medium-gpu", Resources: PodVMResources{GPUs: 2, VCPUs: 4, Memory: 8192}}, }, gpus: 1, vcpus: 2, @@ -559,8 +637,8 @@ func TestGetBestFitInstanceTypeWithGPU(t *testing.T) { { name: "next best fit", specList: []InstanceTypeSpec{ - {InstanceType: "small-gpu", GPUs: 1, VCPUs: 2, Memory: 4096}, - {InstanceType: "medium-gpu", GPUs: 2, VCPUs: 4, Memory: 8192}, + {InstanceType: "small-gpu", Resources: PodVMResources{GPUs: 1, VCPUs: 2, Memory: 4096}}, + {InstanceType: "medium-gpu", Resources: PodVMResources{GPUs: 2, VCPUs: 4, Memory: 8192}}, }, gpus: 1, vcpus: 3, @@ -571,8 +649,8 @@ func TestGetBestFitInstanceTypeWithGPU(t *testing.T) { { name: "no match found", specList: []InstanceTypeSpec{ - {InstanceType: "small-gpu", GPUs: 1, VCPUs: 2, Memory: 4096}, - {InstanceType: "medium-gpu", GPUs: 2, VCPUs: 4, Memory: 8192}, + {InstanceType: "small-gpu", Resources: PodVMResources{GPUs: 1, VCPUs: 2, Memory: 4096}}, + {InstanceType: "medium-gpu", Resources: PodVMResources{GPUs: 2, VCPUs: 4, Memory: 8192}}, }, gpus: 4, vcpus: 8,