Skip to content

Commit

Permalink
Implement support for configurable Spot allocation strategies
Browse files Browse the repository at this point in the history 
- Added global config with per-ASG tag overrides
- Extended unit test coverage for the new logic
- Improved tests for reading other configurations from tags
- Added unit tests for EBS block device conversion logic
- Converted PatchBeanstalkUserdata config flag to bool value
  • Loading branch information
@cristim
cristim committed Sep 16, 2021
1 parent c845477 commit b576743
Show file tree
Hide file tree
Showing 11 changed files with 1,218 additions and 101 deletions.
9 changes: 4 additions & 5 deletions core/autoscaling.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ type autoScalingGroup struct {
launchConfiguration *launchConfiguration
launchTemplate *launchTemplate
instances instances
minOnDemand int64
config AutoScalingConfig
}

Expand Down Expand Up @@ -121,20 +120,20 @@ func (a *autoScalingGroup) loadLaunchTemplate() (*launchTemplate, error) {
func (a *autoScalingGroup) needReplaceOnDemandInstances() (bool, int64) {
onDemandRunning, totalRunning := a.alreadyRunningInstanceCount(false, nil)
debug.Printf("onDemandRunning=%v totalRunning=%v a.minOnDemand=%v",
onDemandRunning, totalRunning, a.minOnDemand)
onDemandRunning, totalRunning, a.config.MinOnDemand)

if totalRunning == 0 {
log.Printf("The group %s is currently empty or in the process of launching new instances",
a.name)
return true, totalRunning
}

if onDemandRunning > a.minOnDemand {
if onDemandRunning > a.config.MinOnDemand {
log.Println("Currently more than enough OnDemand instances running")
return true, totalRunning
}

if onDemandRunning == a.minOnDemand {
if onDemandRunning == a.config.MinOnDemand {
log.Println("Currently OnDemand running equals to the required number, skipping run")
return false, totalRunning
}
Expand All @@ -150,7 +149,7 @@ func (a *autoScalingGroup) terminateRandomSpotInstanceIfHavingEnough(totalRunnin
}

if allInstancesAreRunning, onDemandRunning := a.allInstancesRunning(); allInstancesAreRunning {
if a.instances.count64() == *a.DesiredCapacity && onDemandRunning == a.minOnDemand {
if a.instances.count64() == *a.DesiredCapacity && onDemandRunning == a.config.MinOnDemand {
log.Println("Currently Spot running equals to the required number, skipping termination")
return nil
}
Expand Down
139 changes: 98 additions & 41 deletions core/autoscaling_configuration.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,16 @@ const (
// GP2ConversionThresholdTag is the name of the tag set on the AutoScaling Group that
// can override the global value of the GP2ConversionThreshold parameter
GP2ConversionThresholdTag = "autospotting_gp2_conversion_threshold"

// SpotAllocationStrategyTag is the name of the tag set on the AutoScaling Group that
// can override the global value of the SpotAllocationStrategy parameter
SpotAllocationStrategyTag = "autospotting_spot_allocation_strategy"
)

// AutoScalingConfig stores some group-specific configurations that can override
// their corresponding global values
type AutoScalingConfig struct {
MinOnDemand int64
MinOnDemandNumber int64
MinOnDemandPercentage float64
AllowedInstanceTypes string
Expand All @@ -136,10 +141,16 @@ type AutoScalingConfig struct {
CronTimezone string
CronScheduleState string // "on" or "off", dictate whether to run inside the CronSchedule or not

PatchBeanstalkUserdata string
PatchBeanstalkUserdata bool

// Threshold for converting EBS volumes from GP2 to GP3, since after a certain size GP2 may be more performant than GP3.
// Threshold for converting EBS volumes from GP2 to GP3, since after a certain
// size GP2 may be more performant than GP3.
GP2ConversionThreshold int64

// Controls the instance type selection when launching new Spot instances.
// Further information about this is available at
// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-fleet-allocation-strategy.html
SpotAllocationStrategy string
}

func (a *autoScalingGroup) loadPercentageOnDemand(tagValue *string) (int64, bool) {
Expand Down Expand Up @@ -214,8 +225,8 @@ func (a *autoScalingGroup) getTagValue(keyMatch string) *string {
}

func (a *autoScalingGroup) setMinOnDemandIfLarger(newValue int64, hasMinOnDemand bool) bool {
if !hasMinOnDemand || newValue > a.minOnDemand {
a.minOnDemand = newValue
if !hasMinOnDemand || newValue > a.config.MinOnDemand {
a.config.MinOnDemand = newValue
}
return true
}
Expand All @@ -241,40 +252,61 @@ func (a *autoScalingGroup) loadConfOnDemand() bool {
return foundLimit
}

func (a *autoScalingGroup) loadPatchBeanstalkUserdata() {
func (a *autoScalingGroup) loadPatchBeanstalkUserdata() bool {
tagValue := a.getTagValue(PatchBeanstalkUserdataTag)

if tagValue != nil {
log.Printf("Loaded PatchBeanstalkUserdata value %v from tag %v\n", *tagValue, PatchBeanstalkUserdataTag)
a.config.PatchBeanstalkUserdata = *tagValue
return
}
val, err := strconv.ParseBool(*tagValue)

if err != nil {
log.Printf("Failed to parse PatchBeanstalkUserdata value %v as a boolean", *tagValue)
return false
}
a.config.PatchBeanstalkUserdata = val
return true
}
debug.Println("Couldn't find tag", PatchBeanstalkUserdataTag, "on the group", a.name, "using the default configuration")
a.config.PatchBeanstalkUserdata = a.region.conf.PatchBeanstalkUserdata
return false
}

func (a *autoScalingGroup) loadSpotAllocationStrategy() bool {
a.config.SpotAllocationStrategy = a.region.conf.SpotAllocationStrategy

tagValue := a.getTagValue(SpotAllocationStrategyTag)

if tagValue != nil {
log.Printf("Loaded AllocationStrategy value %v from tag %v\n", *tagValue, SpotAllocationStrategyTag)
a.config.SpotAllocationStrategy = *tagValue
return true
}

debug.Println("Couldn't find tag", SpotAllocationStrategyTag, "on the group", a.name, "using the default configuration")
return false
}

func (a *autoScalingGroup) loadGP2ConversionThreshold() {
func (a *autoScalingGroup) loadGP2ConversionThreshold() bool {
// setting the default value
a.config.GP2ConversionThreshold = a.region.conf.GP2ConversionThreshold

tagValue := a.getTagValue(GP2ConversionThresholdTag)
if tagValue == nil {
log.Printf("Couldn't load the GP2ConversionThreshold from tag %v, using the globally configured value of %v\n", GP2ConversionThresholdTag, a.config.GP2ConversionThreshold)
return
return false
}

log.Printf("Loaded GP2ConversionThreshold value %v from tag %v\n", *tagValue, GP2ConversionThresholdTag)

threshold, err := strconv.Atoi(*tagValue)
if err != nil {
log.Printf("Error parsing %v qs integer: %s\n", *tagValue, err.Error())
return
return false
}

debug.Println("Successfully parsed", GP2ConversionThresholdTag, "on the group", a.name, "overriding the default configuration")
a.config.GP2ConversionThreshold = int64(threshold)

return true
}

func (a *autoScalingGroup) loadBiddingPolicy(tagValue *string) (string, bool) {
Expand All @@ -287,42 +319,45 @@ func (a *autoScalingGroup) loadBiddingPolicy(tagValue *string) (string, bool) {
return biddingPolicy, true
}

func (a *autoScalingGroup) LoadCronSchedule() {
func (a *autoScalingGroup) LoadCronSchedule() bool {
tagValue := a.getTagValue(ScheduleTag)

if tagValue != nil {
log.Printf("Loaded CronSchedule value %v from tag %v\n", *tagValue, ScheduleTag)
a.config.CronSchedule = *tagValue
return
return true
}

debug.Println("Couldn't find tag", ScheduleTag, "on the group", a.name, "using the default configuration")
a.config.CronSchedule = a.region.conf.CronSchedule
return false
}

func (a *autoScalingGroup) LoadCronTimezone() {
func (a *autoScalingGroup) LoadCronTimezone() bool {
tagValue := a.getTagValue(TimezoneTag)

if tagValue != nil {
log.Printf("Loaded CronTimezone value %v from tag %v\n", *tagValue, TimezoneTag)
a.config.CronTimezone = *tagValue
return
return true
}

debug.Println("Couldn't find tag", TimezoneTag, "on the group", a.name, "using the default configuration")
a.config.CronTimezone = a.region.conf.CronTimezone
return false
}

func (a *autoScalingGroup) LoadCronScheduleState() {
func (a *autoScalingGroup) LoadCronScheduleState() bool {
tagValue := a.getTagValue(CronScheduleStateTag)
if tagValue != nil {
log.Printf("Loaded CronScheduleState value %v from tag %v\n", *tagValue, CronScheduleStateTag)
a.config.CronScheduleState = *tagValue
return
return true
}

debug.Println("Couldn't find tag", CronScheduleStateTag, "on the group", a.name, "using the default configuration")
a.config.CronScheduleState = a.region.conf.CronScheduleState
return false
}

func (a *autoScalingGroup) loadConfSpot() bool {
Expand Down Expand Up @@ -357,7 +392,7 @@ func (a *autoScalingGroup) loadConfSpotPrice() bool {
}

func (a *autoScalingGroup) loadConfOnDemandPriceMultiplier() bool {

a.config.OnDemandPriceMultiplier = a.region.conf.OnDemandPriceMultiplier
tagValue := a.getTagValue(OnDemandPriceMultiplierTag)
if tagValue == nil {
return false
Expand All @@ -375,37 +410,59 @@ func (a *autoScalingGroup) loadConfOnDemandPriceMultiplier() bool {

// Add configuration of other elements here: prices, whitelisting, etc
func (a *autoScalingGroup) loadConfigFromTags() bool {
ret := false

resOnDemandConf := a.loadConfOnDemand()
if a.loadConfOnDemand() {
log.Println("Found and applied configuration for OnDemand value")
ret = true
}

resOnDemandPriceMultiplierConf := a.loadConfOnDemandPriceMultiplier()
if a.loadConfOnDemandPriceMultiplier() {
log.Println("Found and applied configuration for OnDemand Price Multiplier")
ret = true
}

resSpotConf := a.loadConfSpot()
if a.loadConfSpot() {
log.Println("Found and applied configuration for Spot Bid")
ret = true
}

resSpotPriceConf := a.loadConfSpotPrice()
if a.loadConfSpotPrice() {
log.Println("Found and applied configuration for Spot Price")
ret = true
}

a.LoadCronSchedule()
a.LoadCronTimezone()
a.LoadCronScheduleState()
a.loadPatchBeanstalkUserdata()
a.loadGP2ConversionThreshold()
if a.LoadCronSchedule() {
log.Println("Found and applied configuration for Spot Price")
ret = true
}

if resOnDemandConf {
log.Println("Found and applied configuration for OnDemand value")
if a.LoadCronTimezone() {
log.Println("Found and applied configuration for Spot Price")
ret = true
}
if resOnDemandPriceMultiplierConf {
log.Println("Found and applied configuration for OnDemand Price Multiplier")

if a.LoadCronScheduleState() {
log.Println("Found and applied configuration for Spot Price")
ret = true
}
if resSpotConf {
log.Println("Found and applied configuration for Spot Bid")

if a.loadPatchBeanstalkUserdata() {
log.Println("Found and applied configuration for Spot Price")
ret = true
}
if resSpotPriceConf {

if a.loadGP2ConversionThreshold() {
log.Println("Found and applied configuration for Spot Price")
ret = true
}
if resOnDemandConf || resOnDemandPriceMultiplierConf || resSpotConf || resSpotPriceConf {
return true

if a.loadSpotAllocationStrategy() {
log.Println("Found and applied configuration for Spot Price")
ret = true
}
return false

return ret
}

func (a *autoScalingGroup) loadDefaultConfigNumber() (int64, bool) {
Expand All @@ -432,17 +489,17 @@ func (a *autoScalingGroup) loadDefaultConfigPercentage() (int64, bool) {

func (a *autoScalingGroup) loadDefaultConfig() bool {
done := false
a.minOnDemand = DefaultMinOnDemandValue
a.config.MinOnDemand = DefaultMinOnDemandValue

if a.region.conf.SpotPriceBufferPercentage <= 0 {
a.region.conf.SpotPriceBufferPercentage = DefaultSpotPriceBufferPercentage
}

if a.region.conf.MinOnDemandNumber != 0 {
a.minOnDemand, done = a.loadDefaultConfigNumber()
a.config.MinOnDemand, done = a.loadDefaultConfigNumber()
}
if !done && a.region.conf.MinOnDemandPercentage != 0 {
a.minOnDemand, done = a.loadDefaultConfigPercentage()
a.config.MinOnDemand, done = a.loadDefaultConfigPercentage()
} else {
log.Println("No default value for on-demand instances specified, skipping.")
}
Expand Down
Loading

0 comments on commit b576743

Please sign in to comment.