Feat/karpenter reconciler (#2428)

* Setting up the Karpenter reconciler cronjob * Matching both configmaps * Putting configmap in right namespace * Fixing oopsie * Fixing namespaces * Duh * Removing dependency on Gen3, since that was a pain * Fixing name and adding auth * We need the label for network policies * This should work * Update application.yaml * Moving everything over to argo-events namespace * Removed configmap.yaml, and moved everything over to argo-events namespace * Fixing stuff * Fixing it so they can work together * Fixing a typo in the configmap * Username, not "user name"
uc-cdis · Dec 6, 2023 · 59d3e11 · 59d3e11
1 parent 49051c0
commit 59d3e11
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 18 deletions.
diff --git a/kube/services/argo-events/workflows/configmap.yaml b/kube/services/argo-events/workflows/configmap.yaml
@@ -8,7 +8,7 @@ data:
     apiVersion: karpenter.sh/v1alpha5
     kind: Provisioner
     metadata:
-      name: workflow-$WORKFLOW_NAME
+      name: workflow-WORKFLOW_NAME
     spec:
       requirements:
         - key: karpenter.sh/capacity-type
@@ -18,7 +18,7 @@ data:
           operator: In
           values:
           - amd64
-        - key: karpenter.k8s.aws/instance-family
+        - key: node.kubernetes.io/instance-type
           operator: In
           values:
           - c6a.large
@@ -45,15 +45,15 @@ data:
           - m6i.16xlarge
       taints:
         - key: role
-          value: $WORKFLOW_NAME
+          value: WORKFLOW_NAME
           effect: NoSchedule
       labels:
-        role: $WORKFLOW_NAME
+        role: WORKFLOW_NAME
       limits:
         resources:
           cpu: 2000
       providerRef:
-        name: workflow-$WORKFLOW_NAME
+        name: workflow-WORKFLOW_NAME
       # Kill nodes after 30 days to ensure they stay up to date
       ttlSecondsUntilExpired: 2592000
       ttlSecondsAfterEmpty: 10
@@ -62,18 +62,18 @@ data:
     apiVersion: karpenter.k8s.aws/v1alpha1
     kind: AWSNodeTemplate
     metadata:
-      name: workflow-$WORKFLOW_NAME
+      name: workflow-WORKFLOW_NAME
     spec:
       subnetSelector:
-        karpenter.sh/discovery: $ENVIRONMENT
+        karpenter.sh/discovery: ENVIRONMENT
       securityGroupSelector:
-        karpenter.sh/discovery: $ENVIRONMENT-workflow
+        karpenter.sh/discovery: ENVIRONMENT-workflow
       tags:
-        Environment: $ENVIRONMENT
-        Name: eks-$ENVIRONMENT-workflow-karpenter
-        karpenter.sh/discovery: $ENVIRONMENT
-        workflowname: $WORKFLOW_NAME
-        gen3username: $GEN3_USERNAME
+        Environment: ENVIRONMENT
+        Name: eks-ENVIRONMENT-workflow-karpenter
+        karpenter.sh/discovery: ENVIRONMENT
+        workflowname: WORKFLOW_NAME
+        gen3username: GEN3_USERNAME
         gen3service: argo-workflows
         purpose: workflow
       metadataOptions:

diff --git a/kube/services/argo-events/workflows/sensor-created.yaml b/kube/services/argo-events/workflows/sensor-created.yaml
@@ -60,11 +60,11 @@ spec:
                           - "-c"
                           - |
                             if ! kubectl get awsnodetemplate workflow-$WORKFLOW_NAME >/dev/null 2>&1; then
-                              envsubst < /home/manifests/nodetemplate.yaml  | kubectl apply -f - 
+                              sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/"  | kubectl apply -f - 
                             fi
 
                             if ! kubectl get provisioner workflow-$WORKFLOW_NAME >/dev/null 2>&1; then
-                              envsubst < /home/manifests/provisioner.yaml  | kubectl apply -f - 
+                              sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/"  | kubectl apply -f - 
                             fi
                         env:
                           - name: WORKFLOW_NAME

diff --git a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml
@@ -41,7 +41,7 @@ spec:
                     AWSNODETEMPLATE_TEMPLATE="nodetemplate.yaml"
                   fi
 
-                  ENVIRONMENT=$(kubectl get configmap global -o jsonpath="{.data.environment}")
+                  ENVIRONMENT=$(kubectl -n default get configmap global -o jsonpath="{.data.environment}")
 
                   RAW_WORKFLOWS=$(kubectl get workflows -n argo -o yaml)
 
@@ -60,12 +60,13 @@ spec:
 
                     if ! kubectl get awsnodetemplate workflow-$workflow_name >/dev/null 2>&1; then
                       echo "No awsnodetemplate found for ${workflow_name}, creating one"
-                      sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USER_NAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$AWSNODETEMPLATE_TEMPLATE" | kubectl apply -f -
+                      sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$AWSNODETEMPLATE_TEMPLATE" | kubectl apply -f -
                     fi
 
                     if ! kubectl get provisioner workflow-$workflow_name >/dev/null 2>&1; then
                       echo "No provisioner found for ${workflow_name}, creating one"
-                      sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USER_NAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$PROVISIONER_TEMPLATE" | kubectl apply -f -
+                      sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$PROVISIONER_TEMPLATE" | kubectl apply -f -
+
                     fi
                   done
           restartPolicy: OnFailure