Skip to content

Commit

Permalink
instance tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@jmdeal
jmdeal committed Feb 24, 2025
1 parent d90cfa7 commit babc7a2
Show file tree
Hide file tree
Showing 3 changed files with 189 additions and 8 deletions.
63 changes: 56 additions & 7 deletions pkg/fake/ec2api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,10 @@ import (
)

type CapacityPool struct {
CapacityType string
InstanceType string
Zone string
CapacityType string
InstanceType string
Zone string
ReservationID string
}

// EC2Behavior must be reset between tests otherwise tests will
Expand Down Expand Up @@ -127,13 +128,23 @@ func (e *EC2API) CreateFleet(_ context.Context, input *ec2.CreateFleetInput, _ .
return nil, fmt.Errorf("missing launch template name")
}
var instanceIds []string
var skippedPools []CapacityPool
var icedPools []CapacityPool
var reservationExceededPools []CapacityPool
var spotInstanceRequestID *string

if string(input.TargetCapacitySpecification.DefaultTargetCapacityType) == karpv1.CapacityTypeSpot {
spotInstanceRequestID = aws.String(test.RandomName())
}

launchTemplates := map[string]*ec2.CreateLaunchTemplateInput{}
for e.CreateLaunchTemplateBehavior.CalledWithInput.Len() > 0 {
lt := e.CreateLaunchTemplateBehavior.CalledWithInput.Pop()
launchTemplates[*lt.LaunchTemplateName] = lt
}
for _, ltInput := range launchTemplates {
e.CreateLaunchTemplateBehavior.CalledWithInput.Add(ltInput)
}

fulfilled := 0
for _, ltc := range input.LaunchTemplateConfigs {
for _, override := range ltc.Overrides {
Expand All @@ -142,7 +153,7 @@ func (e *EC2API) CreateFleet(_ context.Context, input *ec2.CreateFleetInput, _ .
if pool.InstanceType == string(override.InstanceType) &&
pool.Zone == aws.ToString(override.AvailabilityZone) &&
pool.CapacityType == string(input.TargetCapacitySpecification.DefaultTargetCapacityType) {
skippedPools = append(skippedPools, pool)
icedPools = append(icedPools, pool)
skipInstance = true
return false
}
Expand All @@ -151,7 +162,34 @@ func (e *EC2API) CreateFleet(_ context.Context, input *ec2.CreateFleetInput, _ .
if skipInstance {
continue
}
amiID := aws.String("")
amiID := lo.ToPtr("")
var capacityReservationID *string
if lt, ok := launchTemplates[lo.FromPtr(ltc.LaunchTemplateSpecification.LaunchTemplateName)]; ok {
amiID = lt.LaunchTemplateData.ImageId
if crs := lt.LaunchTemplateData.CapacityReservationSpecification; crs != nil && crs.CapacityReservationPreference == ec2types.CapacityReservationPreferenceCapacityReservationsOnly {
id := crs.CapacityReservationTarget.CapacityReservationId
if id == nil {
panic("received a launch template targeting capacity reservations without a provided ID")
}
capacityReservationID = id
}
}
if capacityReservationID != nil {
if cr, ok := lo.Find(e.DescribeCapacityReservationsOutput.Clone().CapacityReservations, func(cr ec2types.CapacityReservation) bool {
return *cr.CapacityReservationId == *capacityReservationID
}); !ok || *cr.AvailableInstanceCount == 0 {
reservationExceededPools = append(reservationExceededPools, CapacityPool{
InstanceType: string(override.InstanceType),
Zone: lo.FromPtr(override.AvailabilityZone),
CapacityType: karpv1.CapacityTypeReserved,
ReservationID: *capacityReservationID,
})
skipInstance = true
}
}
if skipInstance {
continue
}
if e.CreateLaunchTemplateBehavior.CalledWithInput.Len() > 0 {
lt := e.CreateLaunchTemplateBehavior.CalledWithInput.Pop()
amiID = lt.LaunchTemplateData.ImageId
Expand Down Expand Up @@ -193,7 +231,7 @@ func (e *EC2API) CreateFleet(_ context.Context, input *ec2.CreateFleetInput, _ .
},
},
}}
for _, pool := range skippedPools {
for _, pool := range icedPools {
result.Errors = append(result.Errors, ec2types.CreateFleetError{
ErrorCode: aws.String("InsufficientInstanceCapacity"),
LaunchTemplateAndOverrides: &ec2types.LaunchTemplateAndOverridesResponse{
Expand All @@ -204,6 +242,17 @@ func (e *EC2API) CreateFleet(_ context.Context, input *ec2.CreateFleetInput, _ .
},
})
}
for _, pool := range reservationExceededPools {
result.Errors = append(result.Errors, ec2types.CreateFleetError{
ErrorCode: lo.ToPtr("ReservationCapacityExceeded"),
LaunchTemplateAndOverrides: &ec2types.LaunchTemplateAndOverridesResponse{
Overrides: &ec2types.FleetLaunchTemplateOverrides{
InstanceType: ec2types.InstanceType(pool.InstanceType),
AvailabilityZone: lo.ToPtr(pool.Zone),
},
},
})
}
return result, nil
})
}
Expand Down
15 changes: 14 additions & 1 deletion pkg/providers/capacityreservation/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,20 @@ func (c *availabilityCache) GetAvailableInstanceCount(reservationID string) int
c.mu.RLock()
defer c.mu.RUnlock()
entry, ok := c.cache.Get(reservationID)
return lo.Ternary(ok, entry.(*availabilityCacheEntry).count, 0)
if !ok {
return 0
}
return entry.(*availabilityCacheEntry).count
}

// TODO: Determine better abstraction for setting availability in tests without reconciling the nodeclass controller
func (c *availabilityCache) SetAvailableInstanceCount(reservationID string, count int) {
c.mu.Lock()
defer c.mu.Unlock()
c.cache.SetDefault(reservationID, &availabilityCacheEntry{
count: count,
syncTime: c.clk.Now(),
})
}

func (c *availabilityCache) MarkUnavailable(reservationIDs ...string) {
Expand Down
119 changes: 119 additions & 0 deletions pkg/providers/instance/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@ import (
"sigs.k8s.io/karpenter/pkg/test/v1alpha1"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/ec2"
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/awslabs/operatorpkg/object"
"github.com/samber/lo"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/record"
Expand Down Expand Up @@ -137,6 +139,123 @@ var _ = Describe("InstanceProvider", func() {
Expect(corecloudprovider.IsInsufficientCapacityError(err)).To(BeTrue())
Expect(instance).To(BeNil())
})
It("should return an ICE error when all attempted instance types return a ReservedCapacityReservation error", func() {
const targetReservationID = "cr-m5.large-1a-1"
// Ensure that Karpenter believes a reservation is available, but the API returns no capacity when attempting to launch
awsEnv.CapacityReservationProvider.SetAvailableInstanceCount(targetReservationID, 1)
awsEnv.EC2API.DescribeCapacityReservationsOutput.Set(&ec2.DescribeCapacityReservationsOutput{
CapacityReservations: []ec2types.CapacityReservation{
{
AvailabilityZone: lo.ToPtr("test-zone-1a"),
InstanceType: lo.ToPtr("m5.large"),
OwnerId: lo.ToPtr("012345678901"),
InstanceMatchCriteria: ec2types.InstanceMatchCriteriaTargeted,
CapacityReservationId: lo.ToPtr(targetReservationID),
AvailableInstanceCount: lo.ToPtr[int32](0),
State: ec2types.CapacityReservationStateActive,
},
},
})
nodeClass.Status.CapacityReservations = append(nodeClass.Status.CapacityReservations, v1.CapacityReservation{
ID: "cr-m5.large-1a-1",
AvailabilityZone: "test-zone-1a",
InstanceMatchCriteria: string(ec2types.InstanceMatchCriteriaTargeted),
InstanceType: "m5.large",
OwnerID: "012345678901",
})
nodeClaim.Spec.Requirements = append(
nodeClaim.Spec.Requirements,
karpv1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: corev1.NodeSelectorRequirement{
Key: karpv1.CapacityTypeLabelKey,
Operator: corev1.NodeSelectorOpIn,
Values: []string{karpv1.CapacityTypeReserved},
}},
)
ExpectApplied(ctx, env.Client, nodeClaim, nodePool, nodeClass)

instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, nodePool)
Expect(err).ToNot(HaveOccurred())
instance, err := awsEnv.InstanceProvider.Create(ctx, nodeClass, nodeClaim, nil, instanceTypes)
Expect(corecloudprovider.IsInsufficientCapacityError(err)).To(BeTrue())
Expect(instance).To(BeNil())

// Ensure we marked the reservation as unavailable after encountering the error
Expect(awsEnv.CapacityReservationProvider.GetAvailableInstanceCount(targetReservationID)).To(Equal(0))
})
It("should filter compatible reserved offerings such that only one offering per capacity pool is included in the CreateFleet request", func() {
const targetReservationID = "cr-m5.large-1a-2"
awsEnv.EC2API.DescribeCapacityReservationsOutput.Set(&ec2.DescribeCapacityReservationsOutput{
CapacityReservations: []ec2types.CapacityReservation{
{
AvailabilityZone: lo.ToPtr("test-zone-1a"),
InstanceType: lo.ToPtr("m5.large"),
OwnerId: lo.ToPtr("012345678901"),
InstanceMatchCriteria: ec2types.InstanceMatchCriteriaTargeted,
CapacityReservationId: lo.ToPtr("cr-m5.large-1a-1"),
AvailableInstanceCount: lo.ToPtr[int32](1),
State: ec2types.CapacityReservationStateActive,
},
{
AvailabilityZone: lo.ToPtr("test-zone-1a"),
InstanceType: lo.ToPtr("m5.large"),
OwnerId: lo.ToPtr("012345678901"),
InstanceMatchCriteria: ec2types.InstanceMatchCriteriaTargeted,
CapacityReservationId: lo.ToPtr(targetReservationID),
AvailableInstanceCount: lo.ToPtr[int32](2),
State: ec2types.CapacityReservationStateActive,
},
},
})
awsEnv.CapacityReservationProvider.SetAvailableInstanceCount("cr-m5.large-1a-1", 1)
awsEnv.CapacityReservationProvider.SetAvailableInstanceCount(targetReservationID, 2)
nodeClass.Status.CapacityReservations = append(nodeClass.Status.CapacityReservations, []v1.CapacityReservation{
{
ID: "cr-m5.large-1a-1",
AvailabilityZone: "test-zone-1a",
InstanceMatchCriteria: string(ec2types.InstanceMatchCriteriaTargeted),
InstanceType: "m5.large",
OwnerID: "012345678901",
},
{
ID: "cr-m5.large-1a-2",
AvailabilityZone: "test-zone-1a",
InstanceMatchCriteria: string(ec2types.InstanceMatchCriteriaTargeted),
InstanceType: "m5.large",
OwnerID: "012345678901",
},
}...)

nodeClaim.Spec.Requirements = append(
nodeClaim.Spec.Requirements,
karpv1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: corev1.NodeSelectorRequirement{
Key: karpv1.CapacityTypeLabelKey,
Operator: corev1.NodeSelectorOpIn,
Values: []string{karpv1.CapacityTypeReserved},
}},
)
ExpectApplied(ctx, env.Client, nodeClaim, nodePool, nodeClass)

instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, nodePool)
Expect(err).ToNot(HaveOccurred())
instance, err := awsEnv.InstanceProvider.Create(ctx, nodeClass, nodeClaim, nil, instanceTypes)
Expect(err).ToNot(HaveOccurred())
Expect(instance.CapacityType).To(Equal(karpv1.CapacityTypeReserved))
Expect(instance.CapacityReservationID).To(Equal(targetReservationID))

// We should have only created a single launch template, for the single capacity reservation we're attempting to launch
var launchTemplates []*ec2.CreateLaunchTemplateInput
for awsEnv.EC2API.CreateLaunchTemplateBehavior.CalledWithInput.Len() > 0 {
launchTemplates = append(launchTemplates, awsEnv.EC2API.CreateLaunchTemplateBehavior.CalledWithInput.Pop())
}
Expect(launchTemplates).To(HaveLen(1))
Expect(*launchTemplates[0].LaunchTemplateData.CapacityReservationSpecification.CapacityReservationTarget.CapacityReservationId).To(Equal(targetReservationID))

Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).ToNot(Equal(0))
createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop()
Expect(createFleetInput.TargetCapacitySpecification.DefaultTargetCapacityType).To(Equal(ec2types.DefaultTargetCapacityTypeOnDemand))
Expect(createFleetInput.LaunchTemplateConfigs).To(HaveLen(1))
Expect(createFleetInput.LaunchTemplateConfigs[0].Overrides).To(HaveLen(1))
})
It("should return all NodePool-owned instances from List", func() {
ids := sets.New[string]()
// Provision instances that have the karpenter.sh/nodepool key
Expand Down

0 comments on commit babc7a2

Please sign in to comment.