Move user scheduler to general and add tags and lables to asg and nod…

…e groups to support scaling from zero.

src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf

@@ -39,6 +39,10 @@ resource "aws_eks_node_group" "main" {
     max_size     = var.node_groups[count.index].max_size
   }
 
+  labels = {
+    "dedicated" = var.node_groups[count.index].name
+  }
+
   lifecycle {
     ignore_changes = [
       scaling_config[0].desired_size,
@@ -54,9 +58,40 @@ resource "aws_eks_node_group" "main" {
 
   tags = merge({
     "kubernetes.io/cluster/${var.name}" = "shared"
+    "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name
+    propagate_at_launch = true
   }, var.tags)
 }
 
+resource "aws_autoscaling_group_tag" "dedicated_user" {
+  for_each = toset(
+    [for asg in flatten(
+      [for resources in aws_eks_node_group.main[1].resources : resources.autoscaling_groups]
+    ) : asg.name]
+  )
+  autoscaling_group_name = each.value
+  tag {
+    key   = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value = var.node_groups[1].name
+    propagate_at_launch = true
+  }
+}
+
+resource "aws_autoscaling_group_tag" "dedicated_worker" {
+  for_each = toset(
+    [for asg in flatten(
+      [for resources in aws_eks_node_group.main[2].resources : resources.autoscaling_groups]
+    ) : asg.name]
+  )
+  autoscaling_group_name = each.value
+  tag {
+    key   = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value = var.node_groups[2].name
+    propagate_at_launch = true
+  }
+}
+
+
 data "aws_eks_cluster_auth" "main" {
   name = aws_eks_cluster.main.name
 }

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf

@@ -134,14 +134,14 @@ resource "helm_release" "jupyterhub" {
       singleuser = {
         image = var.jupyterlab-image
         nodeSelector = {
-          "${var.user-node-group.key}" = var.user-node-group.value
+          "dedicated" = var.user-node-group.value
         }
       }
 
       scheduling = {
         userScheduler = {
           nodeSelector = {
-            "${var.user-node-group.key}" = var.user-node-group.value
+            "${var.user-node-group.key}" = var.general-node-group.value
           }
         }
       }