Skip to content

Commit

Permalink
predefined machine type
Browse files Browse the repository at this point in the history
  • Loading branch information
THWiseman committed Nov 3, 2023
1 parent 110ca3e commit 5a381db
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,21 @@ object MachineConstraints {
): String = {
if (googleLegacyMachineSelection) {
s"predefined-$cpu-${memory.to(MemoryUnit.MB).amount.intValue()}"
}
else if (cpuPlatformOption.contains(PipelinesApiRuntimeAttributes.CpuPlatformIntelSapphireRapidsValue)) {
// As of 11/2023, C3 machine types (the general purpose platform for the Intel Sapphire Rapids CPU)
// cannot be customized like other general purpose platforms can. Select one of a few legal options.
PredefinedMachineType.getClosestC3Machine(memory, cpu, jobLogger)
} else {
// If someone requests Intel Cascade Lake as their CPU platform then switch the machine type to n2.
// Similarly, CPU platform of AMD Rome corresponds to the machine type n2d.
// Users specify a CPU platform in their WDL, but GCP also needs to know which machine type to use.
// The below logic infers the machine type from the requested CPU. We're assuming that users want the newest
// "General Purpose" machine type that is compatible with the requested CPU. For example, if someone requests
// Intel Cascade Lake as their CPU platform, then infer the n2 machine type. AMD Rome -> n2d.
// The heuristic we're using is "find the newest 'General Purpose' type that supports the given CPU.
// https://cloud.google.com/compute/docs/machine-resource
val customMachineType =
cpuPlatformOption match {
case Some(PipelinesApiRuntimeAttributes.CpuPlatformIntelIceLakeValue) => N2CustomMachineType
case Some(PipelinesApiRuntimeAttributes.CpuPlatformIntelCascadeLakeValue) => N2CustomMachineType
case Some(PipelinesApiRuntimeAttributes.CpuPlatformAMDRomeValue) => N2DCustomMachineType
case _ => N1CustomMachineType
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ object PipelinesApiRuntimeAttributes {
// via `gcloud compute zones describe us-central1-a`
val CpuPlatformIntelCascadeLakeValue = "Intel Cascade Lake"
val CpuPlatformAMDRomeValue = "AMD Rome"
val CpuPlatformIntelIceLakeValue = "Intel Ice Lake"
val CpuPlatformIntelSapphireRapidsValue = "Intel Sapphire Rapids"

val UseDockerImageCacheKey = "useDockerImageCache"
private val useDockerImageCacheValidationInstance = new BooleanRuntimeAttributesValidation(UseDockerImageCacheKey).optional
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package cromwell.backend.google.pipelines.common

import cromwell.backend.google.pipelines.common.CustomMachineType._
import eu.timepit.refined.api.Refined
import eu.timepit.refined.numeric.Positive
import eu.timepit.refined.refineV
import mouse.all._
import org.slf4j.Logger
import wdl4s.parser.MemoryUnit
import wom.format.MemorySize
import math.{log, pow}

case class PredefinedMachineType(cpuCount: Int, gcpString: String)
object PredefinedMachineType {
val c3Standard_4 = PredefinedMachineType(4, "c3-standard-4")
val c3Standard_8 = PredefinedMachineType(8, "c3-standard-8")
val c3Standard_22 = PredefinedMachineType(22, "c3-standard-22")
val c3Standard_44 = PredefinedMachineType(44, "c3-standard-44")
val c3Standard_88 = PredefinedMachineType(88, "c3-standard-88")
val c3Standard_176 = PredefinedMachineType(176, "c3-standard-176")

def getClosestC3Machine(requestedMemory: MemorySize, requestedCpu: Refined[Int, Positive], jobLogger: Logger): String = {
val adjustedMemory: MemorySize = MemorySize(requestedCpu.value * 4.0, MemoryUnit.GB)
if (adjustedMemory != requestedMemory) {
jobLogger.info(s"Adjusting memory from ${requestedMemory.amount} to ${adjustedMemory.amount} in order to match GCP requirements for the requested CPU.")
}
val machine = requestedCpu.value match {
case cpu if cpu <= 4 => c3Standard_4
case cpu if cpu > 4 && cpu <= 8 => c3Standard_8
case cpu if cpu > 8 && cpu <= 22 => c3Standard_22
case cpu if cpu > 22 && cpu <= 44 => c3Standard_44
case cpu if cpu > 44 && cpu <= 88 => c3Standard_88
case cpu if cpu > 88 => c3Standard_176
}

if(machine.cpuCount != requestedCpu.value) {
jobLogger.info(s"Rounding up CPU count from ${requestedCpu} to ${machine.cpuCount} in order to match GCP requirements for the requested CPU.")
}
machine.gcpString
}
}

0 comments on commit 5a381db

Please sign in to comment.