Feat/es proxy irsa no terragrunt

gen3/bin/kube-setup-aws-es-proxy.sh

@@ -12,13 +12,13 @@ gen3_load "gen3/lib/kube-setup-init"
 manifestPath=$(g3k_manifest_path)
 es7="$(jq -r ".[\"global\"][\"es7\"]" < "$manifestPath" | tr '[:upper:]' '[:lower:]')"
 esDomain="$(jq -r ".[\"global\"][\"esDomain\"]" < "$manifestPath" | tr '[:upper:]' '[:lower:]')"
+envname="$(gen3 api environment)"
 
 [[ -z "$GEN3_ROLL_ALL" ]] && gen3 kube-setup-secrets
 
 if g3kubectl get secrets/aws-es-proxy > /dev/null 2>&1; then
-  envname="$(gen3 api environment)"
   if [ "$esDomain" != "null" ]; then
-    if ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names ${esDomain}  --query "DomainStatusList[*].Endpoints" --output text)" \
+    if ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names "${esDomain}"  --query "DomainStatusList[*].Endpoints" --output text)" \
         && [[ -n "${ES_ENDPOINT}" && -n "${esDomain}" ]]; then
       gen3 roll aws-es-proxy GEN3_ES_ENDPOINT "${ES_ENDPOINT}"
            g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-priority-class.yaml"
@@ -34,10 +34,10 @@ if g3kubectl get secrets/aws-es-proxy > /dev/null 2>&1; then
       g3kubectl patch deployment "aws-es-proxy-deployment" -p  '{"spec":{"template":{"metadata":{"labels":{"netvpc":"yes"}}}}}' || true
     fi
   elif [ "$es7" = true ]; then
-    if ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names ${envname}-gen3-metadata-2 --query "DomainStatusList[*].Endpoints" --output text)" \
+    if ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names "${envname}"-gen3-metadata-2 --query "DomainStatusList[*].Endpoints" --output text)" \
         && [[ -n "${ES_ENDPOINT}" && -n "${envname}" ]]; then
       gen3 roll aws-es-proxy GEN3_ES_ENDPOINT "${ES_ENDPOINT}"
-           g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-priority-class.yaml"
+      g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-priority-class.yaml"
       g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-service.yaml"
       gen3_log_info "kube-setup-aws-es-proxy" "The aws-es-proxy service has been deployed onto the k8s cluster."
     else
@@ -50,7 +50,7 @@ if g3kubectl get secrets/aws-es-proxy > /dev/null 2>&1; then
       g3kubectl patch deployment "aws-es-proxy-deployment" -p  '{"spec":{"template":{"metadata":{"labels":{"netvpc":"yes"}}}}}' || true
     fi
   else
-    if ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names ${envname}-gen3-metadata --query "DomainStatusList[*].Endpoints" --output text)" \
+    if ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names "${envname}"-gen3-metadata --query "DomainStatusList[*].Endpoints" --output text)" \
         && [[ -n "${ES_ENDPOINT}" && -n "${envname}" ]]; then
       gen3 roll aws-es-proxy GEN3_ES_ENDPOINT "${ES_ENDPOINT}"
       g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-service.yaml"
@@ -67,6 +67,76 @@ if g3kubectl get secrets/aws-es-proxy > /dev/null 2>&1; then
   fi
   gen3 job cron es-garbage '@daily'
 else
-  gen3_log_info "kube-setup-aws-es-proxy"  "Not deploying aws-es-proxy - secret is not configured"
-  exit 1
-fi
+    gen3_log_info "kube-setup-aws-es-proxy" "No secret detected, attempting IRSA setup"
+    deploy=true
+
+    # Let's pre-calculate all the info we need about the cluster, so we can just pass it on later
+    if [ "$esDomain" != "null" ] && [ -n "$esDomain" ]; then
+      ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names "${esDomain}" --query "DomainStatusList[*].Endpoints" --output text)"
+      ES_ARN="$(aws es describe-elasticsearch-domains --domain-names "${esDomain}" --query "DomainStatusList[*].ARN" --output text)"
+    elif [ "$es7" = true ]; then
+      if [ -n "$envname" ]; then
+        ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names "${envname}"-gen3-metadata-2 --query "DomainStatusList[*].Endpoints" --output text)" 
+        ES_ARN="$(aws es describe-elasticsearch-domains --domain-names "${envname}"-gen3-metadata-2 --query "DomainStatusList[*].ARN" --output text)"
+      else
+        deploy=false
+      fi
+    else
+      if [ -n "$envname" ]; then
+        ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names "${envname}"-gen3-metadata --query "DomainStatusList[*].Endpoints" --output text)" 
+        ES_ARN="$(aws es describe-elasticsearch-domains --domain-names "${envname}"-gen3-metadata --query "DomainStatusList[*].ARN" --output text)"
+      else
+        deploy=false
+      fi
+    fi
+     # Let's only do setup stuff if we're going to want to deploy... otherwise, we take the CI env actions
+    if [ "$deploy" = "true" ]; then
+      # Put that ARN into a template we get from terraform
+      policyjson=$(cat <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "es:*",
+      "Effect": "Allow",
+      "Resource": [
+        "$ES_ARN",
+        "${ES_ARN}/*"
+      ]
+    }
+  ]
+}
+POLICY
+)
+
+      # Creating the role
+      roleName="$(gen3 api safe-name es-access)"
+      saName="esproxy-sa"
+      policyName="$(gen3 api safe-name es-access-policy)"
+
+      gen3 awsrole create "$roleName" "$saName"
+      policyArn=$(gen3_aws_run aws iam list-policies --query "Policies[?PolicyName=='$policyName'].Arn" --output text)
+
+      if [ -n "$policyArn" ]; then
+        echo "No need to create policy, it already exists" 
+      else
+        gen3_aws_run aws iam create-policy --policy-name "$policyName" --policy-document "$policyjson" --description "Allow access to the given ElasticSearch cluster"
+      fi 
+
+      # Now we need some info on the policy, so we can attach the role and the plicy
+      policyArn=$(gen3_aws_run aws iam list-policies --query "Policies[?PolicyName=='$policyName'].Arn" --output text)
+      gen3 awsrole attach-policy "${policyArn}" --role-name "${roleName}" --force-aws-cli || exit 1
+
+      g3k_manifest_filter "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-deploy-irsa.yaml" "" GEN3_ES_ENDPOINT "${ES_ENDPOINT}" | g3kubectl apply -f - 
+      # Then we have to do the whole setup... just copy and modify from above
+      if [ "$es7" = true ]; then
+        g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-priority-class.yaml"
+      fi
+      g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-service.yaml"
+      gen3_log_info "kube-setup-aws-es-proxy" "The aws-es-proxy service has been deployed onto the k8s cluster."
+    else
+      gen3_log_info "kube-setup-aws-es-proxy" "Not deploying aws-es-proxy, no endpoint to hook it up."
+      gen3 kube-setup-networkpolicy service aws-es-proxy
+      g3kubectl patch deployment "aws-es-proxy-deployment" -p  '{"spec":{"template":{"metadata":{"labels":{"netvpc":"yes"}}}}}' || true
+    fi
+  fi

kube/services/aws-es-proxy/aws-es-proxy-deploy-irsa.yaml

@@ -0,0 +1,83 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: aws-es-proxy-deployment
+  annotations:
+    gen3.io/network-ingress: "arranger,arranger-server,arranger-dashboard,guppy,metadata,spark,tube"
+spec:
+  selector:
+    # Only select pods based on the 'app' label
+    matchLabels:
+      app: esproxy
+  revisionHistoryLimit: 2
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  template:
+    metadata:
+      labels:
+        app: esproxy
+        netvpc: "yes"
+        GEN3_DATE_LABEL
+        GEN3_HOSTNAME_LABEL
+    spec:
+      serviceAccountName: esproxy-sa
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: karpenter.sh/capacity-type
+                operator: In
+                values:
+                - on-demand
+          - weight: 99
+            preference:
+              matchExpressions:
+              - key: eks.amazonaws.com/capacityType
+                operator: In
+                values:
+                - ONDEMAND
+      automountServiceAccountToken: false
+      priorityClassName: aws-es-proxy-high-priority
+      containers:
+      - name: esproxy
+        GEN3_AWS-ES-PROXY_IMAGE|-image: quay.io/cdis/aws-es-proxy:v1.3.1-|
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 9200
+        env:
+          - name: "ES_ENDPOINT"
+            GEN3_ES_ENDPOINT|-value: es.internal.io-|
+        command: ["/bin/sh"]
+        # NOTE- NEED TO RUN `gen3 kube-setup-aws-es-proxy` TO POPULATE ES_ENDPOINT - ugh!
+        # NOTE- `gen3 roll aws-es-proxy` WILL NOT WORK!
+        args:
+          - "-c"
+          - |
+            if [ -f /aws-es-proxy ];
+            then
+              # 1.3 needs this PR: https://github.com/uc-cdis/aws-es-proxy/pull/2
+              # aws-es-proxy 1.0+ is prone to throw ES timeout error from client
+              # customize timeout value to compensate this, note the -timeout option only works for 1.2+
+              BINARY="/aws-es-proxy -timeout 180"
+            elif [ -f /usr/local/bin/aws-es-proxy ];
+            then
+              # 0.9
+              BINARY=/usr/local/bin/aws-es-proxy
+            elif [ -f /go/src/github.com/abutaha/aws-es-proxy/aws-es-proxy ];
+            then
+              # 0.8
+              BINARY=/go/src/github.com/abutaha/aws-es-proxy/aws-es-proxy
+            fi
+            ${BINARY} -endpoint "https://$ES_ENDPOINT" -verbose -listen ":9200"
+        resources:
+          requests:
+            cpu: 250m
+            memory: 256Mi
+          limits:
+            cpu: 1000m
+            memory: 2Gi