feat: Consider GCP preemptible VMs as spot VMs (#14)

.github/workflows/ci.yaml

@@ -54,8 +54,8 @@ jobs:
     - name: Login to Docker Hub
       uses: docker/login-action@v3
       with:
-        # We use a GitHub variable to store the Docker Hub username to avoid outputs being skipped:
-        # https://docs.github.com/en/actions/learn-github-actions/variables
+        # We use a GitHub variable to store the Docker Hub username to avoid outputs being skipped
+        # for containing secrets: https://docs.github.com/en/actions/learn-github-actions/variables
         username: ${{ vars.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
     - name: Build and push
@@ -99,6 +99,6 @@ jobs:
         helm template ./charts/cost-manager \
           -n cost-manager \
           --set image.repository="${IMAGE_NAME}" \
-          --set iam.gcp.serviceAccount=cost-manager@example.iam.gserviceaccount.com \
+          --set serviceAccount.annotations."iam\.gke\.io/gcp-service-account"=cost-manager@example.iam.gserviceaccount.com \
           --set vpa.enabled=true | kubectl apply -f -
         kubectl wait --for=condition=Available=true deployment/cost-manager -n cost-manager --timeout=10m

README.md

@@ -19,15 +19,16 @@ workloads are already running on on-demand VMs there is no reason for them to mi
 
 To improve spot VM utilisation, [spot-migrator](./pkg/controller/spot_migrator.go) periodically
 attempts to migrate workloads from on-demand VMs to spot VMs by draining on-demand Nodes to force
-cluster scale up and relying on the fact that the cluster autoscaler [attempts to expand the least
+cluster scale up, relying on the fact that the cluster autoscaler [attempts to expand the least
 expensive possible node
-group](https://github.com/kubernetes/autoscaler/blob/600cda52cf764a1f08b06fc8cc29b1ef95f13c76/cluster-autoscaler/proposals/pricing.md).
-If an on-demand VM is added to the cluster then spot-migrator assumes that there are currently no
-more spot VMs available and waits for the next migration attempt (currently every hour) however if
-no on-demand VMs were added then spot-migrator continues to drain on-demand VMs until there are no
-more left in the cluster (and all workloads are running on spot VMs). Node draining respects [Pod
-Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) to ensure that
-workloads are migrated whilst maintaining desired levels of availability.
+group](https://github.com/kubernetes/autoscaler/blob/600cda52cf764a1f08b06fc8cc29b1ef95f13c76/cluster-autoscaler/proposals/pricing.md),
+taking into account the reduced cost of spot VMs. If an on-demand VM is added to the cluster then
+spot-migrator assumes that there are currently no more spot VMs available and waits for the next
+migration attempt (currently every hour) however if no on-demand VMs were added then spot-migrator
+continues to drain on-demand VMs until there are no more left in the cluster (and all workloads are
+running on spot VMs). Node draining respects [Pod Disruption
+Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) to ensure that workloads
+are migrated whilst maintaining desired levels of availability.
 
 Currently only [GKE
 Standard](https://cloud.google.com/kubernetes-engine/docs/concepts/types-of-clusters) clusters are

charts/cost-manager/templates/service-account.yaml

@@ -3,7 +3,7 @@ kind: ServiceAccount
 metadata:
   name: cost-manager
   namespace: {{ .Release.Namespace }}
-  {{- if .Values.iam.gcp.serviceAccount }}
+{{- if .Values.serviceAccount.annotations }}
   annotations:
-    iam.gke.io/gcp-service-account: {{ .Values.iam.gcp.serviceAccount }}
-  {{- end }}
+{{ .Values.serviceAccount.annotations | toYaml | indent 4 }}
+{{- end }}

charts/cost-manager/values.yaml

@@ -4,12 +4,8 @@ image:
   repository: docker.io/dippynark/cost-manager
   tag: ""
 
-# Specify GCP Workload Identity service account email address bound to the
-# roles/compute.instanceAdmin role to allow compute instance deletion:
-# https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity 
-iam:
-  gcp:
-    serviceAccount: ""
+serviceAccount:
+  annotations: {}
 
 # Create VPA to vertically autoscale cost-manager:
 # https://cloud.google.com/kubernetes-engine/docs/concepts/verticalpodautoscaler

pkg/cloudprovider/gcp/cloud_provider.go

@@ -15,8 +15,9 @@ import (
 
 const (
 	// https://cloud.google.com/kubernetes-engine/docs/concepts/spot-vms#scheduling-workloads
-	spotVMLabelKey   = "cloud.google.com/gke-spot"
-	spotVMLabelValue = "true"
+	spotNodeLabelKey = "cloud.google.com/gke-spot"
+	// https://cloud.google.com/kubernetes-engine/docs/how-to/preemptible-vms#use_nodeselector_to_schedule_pods_on_preemptible_vms
+	preemptibleNodeLabelKey = "cloud.google.com/gke-preemptible"
 
 	// After kube-proxy starts failing its health check GCP load balancers should mark the instance
 	// as unhealthy within 24 seconds but we wait for slightly longer to give in-flight connections
@@ -189,14 +190,12 @@ func (gcp *CloudProvider) DeleteInstance(ctx context.Context, node *corev1.Node)
 	return nil
 }
 
-// IsSpotInstance determines whether the underlying compute instance is a spot VM
+// IsSpotInstance determines whether the underlying compute instance is a spot VM. We consider
+// preemptible VMs to be spot VMs to align with the cluster autoscaler:
+// https://github.com/kubernetes/autoscaler/blob/10fafe758c118adeb55b28718dc826511cc5ba40/cluster-autoscaler/cloudprovider/gce/gce_price_model.go#L220-L230
 func (gcp *CloudProvider) IsSpotInstance(ctx context.Context, node *corev1.Node) (bool, error) {
 	if node.Labels == nil {
 		return false, nil
 	}
-	value, ok := node.Labels[spotVMLabelKey]
-	if !ok {
-		return false, nil
-	}
-	return value == spotVMLabelValue, nil
+	return node.Labels[spotNodeLabelKey] == "true" || node.Labels[preemptibleNodeLabelKey] == "true", nil
 }

pkg/cloudprovider/gcp/cloud_provider_test.go

@@ -14,7 +14,7 @@ func TestIsSpotInstance(t *testing.T) {
 		node           *corev1.Node
 		isSpotInstance bool
 	}{
-		"hasSpotVMLabelSetToTrue": {
+		"hasSpotLabelSetToTrue": {
 			node: &corev1.Node{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: map[string]string{
@@ -24,7 +24,17 @@ func TestIsSpotInstance(t *testing.T) {
 			},
 			isSpotInstance: true,
 		},
-		"hasSpotVMLabelSetToFalse": {
+		"hasPreemptibleLabelSetToTrue": {
+			node: &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"cloud.google.com/gke-preemptible": "true",
+					},
+				},
+			},
+			isSpotInstance: true,
+		},
+		"hasSpotLabelSetToFalse": {
 			node: &corev1.Node{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: map[string]string{
@@ -34,6 +44,16 @@ func TestIsSpotInstance(t *testing.T) {
 			},
 			isSpotInstance: false,
 		},
+		"hasPreemptibleLabelSetToFalse": {
+			node: &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"cloud.google.com/gke-preemptible": "false",
+					},
+				},
+			},
+			isSpotInstance: false,
+		},
 		"hasOtherLabel": {
 			node: &corev1.Node{
 				ObjectMeta: metav1.ObjectMeta{